From e3cc49abb770d3194d26a5c67c38c8b03d04ac9b Mon Sep 17 00:00:00 2001 From: pasha Date: Mon, 23 Oct 2023 17:20:53 +0300 Subject: [PATCH] feat: Add public API prototype, add grpc-web --- Cargo.toml | 2 +- summa-core/src/components/fruit_extractors.rs | 8 +- .../src/components/query_parser/summa_ql.rs | 2 +- summa-core/src/validators.rs | 26 ++++-- summa-proto/Cargo.toml | 2 +- summa-proto/build.rs | 1 + summa-proto/proto/public_service.proto | 12 +++ summa-proto/proto/query.proto | 2 + summa-proto/src/proto_traits/collector.rs | 2 + summa-server/src/apis/index.rs | 2 +- summa-server/src/apis/mod.rs | 1 + summa-server/src/apis/public.rs | 43 ++++++++++ summa-server/src/services/api.rs | 10 ++- summa-server/src/services/index.rs | 43 +++++++++- summa-wasm/Cargo.toml | 2 +- summa-wasm/build.sh | 3 +- summa-wasm/crate/web_index_registry.rs | 10 +++ summa-wasm/package.json | 2 +- .../src/grpc-web/consumer_service.client.ts | 2 +- summa-wasm/src/grpc-web/consumer_service.ts | 2 +- summa-wasm/src/grpc-web/dag_pb.ts | 8 +- summa-wasm/src/grpc-web/index.ts | 4 + .../src/grpc-web/index_service.client.ts | 2 +- summa-wasm/src/grpc-web/index_service.ts | 80 +++++++++--------- .../src/grpc-web/public_service.client.ts | 45 ++++++++++ summa-wasm/src/grpc-web/public_service.ts | 12 +++ summa-wasm/src/grpc-web/query.ts | 82 ++++++++++++------- .../src/grpc-web/reflection_service.client.ts | 2 +- summa-wasm/src/grpc-web/reflection_service.ts | 2 +- .../src/grpc-web/search_service.client.ts | 2 +- summa-wasm/src/grpc-web/search_service.ts | 2 +- summa-wasm/src/grpc-web/unixfs.ts | 28 +++---- summa-wasm/src/grpc-web/utils.ts | 2 +- summa-wasm/src/index-registry.ts | 4 + summa-wasm/src/index.ts | 1 + summa-wasm/src/remote-index-registry.ts | 4 + summa-wasm/src/service-worker.ts | 5 ++ summa-wasm/src/utils.ts | 2 +- 38 files changed, 348 insertions(+), 116 deletions(-) create mode 100644 summa-proto/proto/public_service.proto create mode 100644 summa-server/src/apis/public.rs create mode 100644 summa-wasm/src/grpc-web/index.ts create mode 100644 summa-wasm/src/grpc-web/public_service.client.ts create mode 100644 summa-wasm/src/grpc-web/public_service.ts diff --git a/Cargo.toml b/Cargo.toml index 59e89bd0..25847660 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ serde_cbor = "0.11" serde_json = { version = "1.0" } serde_yaml = { version = "0.8" } strfmt = "0.2" -summa-proto = { version = "0.34.0", path = "./summa-proto", default_features = false } +summa-proto = { version = "0.35.0", path = "./summa-proto", default_features = false } take_mut = "0.2" tantivy = { package = "izihawa-tantivy", version = "0.21.1", default_features = false, features = ["quickwit", "zstd-compression"] } tantivy-common = { package = "izihawa-tantivy-common", version = "0.6.0" } diff --git a/summa-core/src/components/fruit_extractors.rs b/summa-core/src/components/fruit_extractors.rs index e882bc02..e31c57ae 100644 --- a/summa-core/src/components/fruit_extractors.rs +++ b/summa-core/src/components/fruit_extractors.rs @@ -99,7 +99,7 @@ pub fn build_fruit_extractor( ) -> SummaResult> { match collector_proto.collector { Some(proto::collector::Collector::TopDocs(top_docs_collector_proto)) => { - let query_fields = validators::parse_fields(searcher.schema(), &top_docs_collector_proto.fields)?; + let query_fields = validators::parse_fields(searcher.schema(), &top_docs_collector_proto.fields, &top_docs_collector_proto.removed_fields)?; let query_fields = (!query_fields.is_empty()).then(|| HashSet::from_iter(query_fields.into_iter().map(|x| x.0))); Ok(match top_docs_collector_proto.scorer { None | Some(proto::Scorer { scorer: None }) => Box::new( @@ -164,7 +164,11 @@ pub fn build_fruit_extractor( }) } Some(proto::collector::Collector::ReservoirSampling(reservoir_sampling_collector_proto)) => { - let query_fields = validators::parse_fields(searcher.schema(), &reservoir_sampling_collector_proto.fields)?; + let query_fields = validators::parse_fields( + searcher.schema(), + &reservoir_sampling_collector_proto.fields, + &reservoir_sampling_collector_proto.removed_fields, + )?; let query_fields = (!query_fields.is_empty()).then(|| HashSet::from_iter(query_fields.into_iter().map(|x| x.0))); let reservoir_sampling_collector = collectors::ReservoirSampling::with_limit(reservoir_sampling_collector_proto.limit as usize); Ok(Box::new( diff --git a/summa-core/src/components/query_parser/summa_ql.rs b/summa-core/src/components/query_parser/summa_ql.rs index ed96f146..decbba20 100644 --- a/summa-core/src/components/query_parser/summa_ql.rs +++ b/summa-core/src/components/query_parser/summa_ql.rs @@ -190,7 +190,7 @@ impl QueryParser { morphology_manager: &MorphologyManager, tokenizer_manager: &TokenizerManager, ) -> SummaResult { - validators::parse_fields(&schema, &query_parser_config.0.default_fields)?; + validators::parse_fields(&schema, &query_parser_config.0.default_fields, &[])?; Ok(QueryParser { term_field_mappers_manager: TermFieldMappersManager::new(&schema, tokenizer_manager), morphology_manager: morphology_manager.clone(), diff --git a/summa-core/src/validators.rs b/summa-core/src/validators.rs index c6d28f61..d6448e14 100644 --- a/summa-core/src/validators.rs +++ b/summa-core/src/validators.rs @@ -6,9 +6,25 @@ pub fn parse_schema(schema: &str) -> SummaResult { serde_yaml::from_str(schema).map_err(|_| Error::Validation(Box::new(ValidationError::InvalidSchema(schema.to_owned())))) } -pub fn parse_fields<'a>(schema: &'a Schema, fields: &'a [String]) -> SummaResult> { - Ok(fields - .iter() - .map(|f| schema.find_field(f).ok_or_else(|| ValidationError::MissingField(f.to_string()))) - .collect::>()?) +pub fn parse_fields<'a>(schema: &'a Schema, fields: &'a [String], removed_fields: &'a [String]) -> SummaResult> { + if removed_fields.is_empty() { + Ok(fields + .iter() + .map(|f| schema.find_field(f).ok_or_else(|| ValidationError::MissingField(f.to_string()))) + .collect::>()?) + } else if fields.is_empty() { + Ok(schema + .fields() + .map(|(_, field_entry)| { + schema + .find_field(field_entry.name()) + .ok_or_else(|| ValidationError::MissingField(field_entry.name().to_string())) + }) + .collect::>()?) + } else { + Ok(fields + .iter() + .map(|f| schema.find_field(f).ok_or_else(|| ValidationError::MissingField(f.to_string()))) + .collect::>()?) + } } diff --git a/summa-proto/Cargo.toml b/summa-proto/Cargo.toml index 85e319d1..1d36b228 100644 --- a/summa-proto/Cargo.toml +++ b/summa-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "summa-proto" -version = "0.34.0" +version = "0.35.0" authors = ["Pasha Podolsky "] edition = "2021" license-file = "LICENSE" diff --git a/summa-proto/build.rs b/summa-proto/build.rs index 98b47828..2c51c79f 100644 --- a/summa-proto/build.rs +++ b/summa-proto/build.rs @@ -5,6 +5,7 @@ fn main() -> Result<(), Box> { "proto/dag_pb.proto", "proto/index_service.proto", "proto/query.proto", + "proto/public_service.proto", "proto/reflection_service.proto", "proto/search_service.proto", "proto/unixfs.proto", diff --git a/summa-proto/proto/public_service.proto b/summa-proto/proto/public_service.proto new file mode 100644 index 00000000..e6d234a5 --- /dev/null +++ b/summa-proto/proto/public_service.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package summa.proto; + +import "search_service.proto"; +import "query.proto"; + + +// Searches documents in the stored indices +service PublicApi { + // Make search in Summa + rpc search (SearchRequest) returns (SearchResponse) {} +} diff --git a/summa-proto/proto/query.proto b/summa-proto/proto/query.proto index 5ed44467..af94fae4 100644 --- a/summa-proto/proto/query.proto +++ b/summa-proto/proto/query.proto @@ -211,6 +211,7 @@ message FacetCollectorOutput { message ReservoirSamplingCollector { uint32 limit = 1; repeated string fields = 2; + repeated string removed_fields = 3; } message RandomDocument { @@ -230,6 +231,7 @@ message TopDocsCollector { map snippet_configs = 4; bool explain = 5; repeated string fields = 6; + repeated string removed_fields = 7; } message DocumentsCollectorOutput { diff --git a/summa-proto/src/proto_traits/collector.rs b/summa-proto/src/proto_traits/collector.rs index e550dd78..6c7eaa01 100644 --- a/summa-proto/src/proto_traits/collector.rs +++ b/summa-proto/src/proto_traits/collector.rs @@ -42,6 +42,7 @@ pub mod shortcuts { snippet_configs: HashMap::new(), explain: false, fields: Vec::new(), + removed_fields: Vec::new(), })), } } @@ -57,6 +58,7 @@ pub mod shortcuts { snippet_configs: HashMap::new(), explain: false, fields: Vec::new(), + removed_fields: Vec::new(), })), } } diff --git a/summa-server/src/apis/index.rs b/summa-server/src/apis/index.rs index fb8ab2ed..a18f0696 100644 --- a/summa-server/src/apis/index.rs +++ b/summa-server/src/apis/index.rs @@ -152,7 +152,7 @@ impl proto::index_api_server::IndexApi for IndexApiImpl { let schema = index_holder.schema().clone(); let multi_fields = index_holder.multi_fields().clone(); - let query_fields = validators::parse_fields(searcher.schema(), &proto_request.fields).map_err(crate::errors::Error::from)?; + let query_fields = validators::parse_fields(searcher.schema(), &proto_request.fields, &[]).map_err(crate::errors::Error::from)?; let query_fields = (!query_fields.is_empty()).then(|| HashSet::from_iter(query_fields.into_iter().map(|x| x.0))); let documents_receiver = index_holder diff --git a/summa-server/src/apis/mod.rs b/summa-server/src/apis/mod.rs index f66bf89c..6797dccf 100644 --- a/summa-server/src/apis/mod.rs +++ b/summa-server/src/apis/mod.rs @@ -2,5 +2,6 @@ pub mod consumer; pub mod index; +pub mod public; pub mod reflection; pub mod search; diff --git a/summa-server/src/apis/public.rs b/summa-server/src/apis/public.rs new file mode 100644 index 00000000..abfa10a1 --- /dev/null +++ b/summa-server/src/apis/public.rs @@ -0,0 +1,43 @@ +//! Public GRPC API +//! +//! Public GRPC API is using for querying indices but queries are restricted for making it safe to open endpoint in public + +use std::time::Instant; + +use summa_proto::proto; +use tonic::{Request, Response, Status}; +use tracing::{info_span, Instrument}; + +use crate::errors::SummaServerResult; +use crate::services::Index; + +pub struct PublicApiImpl { + index_service: Index, +} + +impl PublicApiImpl { + pub fn new(index_service: &Index) -> SummaServerResult { + Ok(PublicApiImpl { + index_service: index_service.clone(), + }) + } +} + +#[tonic::async_trait] +impl proto::public_api_server::PublicApi for PublicApiImpl { + async fn search(&self, proto_request: Request) -> Result, Status> { + let proto_request = proto_request.into_inner(); + let now = Instant::now(); + let collector_outputs = self + .index_service + .constrained_search(proto_request) + .instrument(info_span!("search")) + .await + .map_err(crate::errors::Error::from)?; + let elapsed_secs = now.elapsed().as_secs_f64(); + Ok(Response::new(proto::SearchResponse { + collector_outputs, + elapsed_secs, + })) + } +} diff --git a/summa-server/src/services/api.rs b/summa-server/src/services/api.rs index 06d273e5..4ff90589 100644 --- a/summa-server/src/services/api.rs +++ b/summa-server/src/services/api.rs @@ -7,6 +7,7 @@ use futures_util::future::try_join_all; use hyper::header::{HeaderName, HeaderValue}; use proto::consumer_api_server::ConsumerApiServer; use proto::index_api_server::IndexApiServer; +use proto::public_api_server::PublicApiServer; use proto::reflection_api_server::ReflectionApiServer; use proto::search_api_server::SearchApiServer; use summa_core::configs::ConfigProxy; @@ -23,6 +24,7 @@ use tracing::{info, info_span, instrument, warn, Instrument, Span}; use crate::apis::consumer::ConsumerApiImpl; use crate::apis::index::IndexApiImpl; +use crate::apis::public::PublicApiImpl; use crate::apis::reflection::ReflectionApiImpl; use crate::apis::search::SearchApiImpl; use crate::errors::SummaServerResult; @@ -77,6 +79,8 @@ impl Api { let index_api = IndexApiImpl::new(&self.server_config_holder, &index_service)?; let reflection_api = ReflectionApiImpl::new(&index_service)?; let search_api = SearchApiImpl::new(&index_service)?; + let public_api = PublicApiImpl::new(&index_service)?; + let grpc_reflection_service = tonic_reflection::server::Builder::configure() .include_reflection_service(false) .register_encoded_file_descriptor_set(proto::FILE_DESCRIPTOR_SET) @@ -121,6 +125,10 @@ impl Api { .max_encoding_message_size(max_from_size_bytes as usize); } + let public_service = PublicApiServer::new(public_api) + .accept_compressed(CompressionEncoding::Gzip) + .send_compressed(CompressionEncoding::Gzip); + let grpc_router = Server::builder() .layer(layer) .max_frame_size(api_config.max_frame_size_bytes.map(|x| x / 256)) @@ -145,7 +153,7 @@ impl Api { })); if let Some(http_endpoint) = api_config.http_endpoint { - let http_router = Server::builder().accept_http1(true).add_service(tonic_web::enable(search_service)); + let http_router = Server::builder().accept_http1(true).add_service(tonic_web::enable(public_service)); let http_listener = Api::set_listener(&http_endpoint)?; let mut http_terminator = terminator.clone(); futures.push(Box::new(async move { diff --git a/summa-server/src/services/index.rs b/summa-server/src/services/index.rs index 8951172a..c0b4ba9d 100644 --- a/summa-server/src/services/index.rs +++ b/summa-server/src/services/index.rs @@ -292,9 +292,9 @@ impl Index { let mut index_attributes = create_index_request.index_attributes.unwrap_or_default(); let query_parser_config = create_index_request.query_parser_config; let default_fields = query_parser_config.as_ref().map(|q| q.default_fields.clone()).unwrap_or_default(); - validators::parse_fields(&schema, &default_fields)?; - validators::parse_fields(&schema, &index_attributes.multi_fields)?; - validators::parse_fields(&schema, &index_attributes.unique_fields)?; + validators::parse_fields(&schema, &default_fields, &[])?; + validators::parse_fields(&schema, &index_attributes.multi_fields, &[])?; + validators::parse_fields(&schema, &index_attributes.unique_fields, &[])?; index_attributes.created_at = SystemTime::now().duration_since(UNIX_EPOCH).expect("cannot retrieve time").as_secs(); @@ -647,6 +647,43 @@ impl Index { Ok(self.index_registry.finalize_extraction(collector_outputs).await?) } + /// Search documents + pub async fn constrained_search(&self, mut search_request: proto::SearchRequest) -> SummaServerResult> { + let index_holder = self.index_registry.get_index_holder(&search_request.index_alias).await?; + let query = search_request + .query + .and_then(|query| query.query) + .unwrap_or_else(|| proto::query::Query::All(proto::AllQuery {})); + + for collector in &mut search_request.collectors { + match &mut collector.collector { + Some(proto::collector::Collector::TopDocs(top_docs)) => { + top_docs.limit = std::cmp::min(top_docs.limit, 10); + top_docs.offset = std::cmp::min(top_docs.offset, 100); + top_docs.removed_fields = vec!["content".to_string()]; + } + Some(proto::collector::Collector::ReservoirSampling(reservoir_sampling)) => { + reservoir_sampling.limit = std::cmp::min(reservoir_sampling.limit, 10); + reservoir_sampling.removed_fields = vec!["content".to_string()]; + } + Some(proto::collector::Collector::Count(_)) => {} + _ => panic!("Not allowed"), + } + } + + let collector_outputs = index_holder + .custom_search( + &search_request.index_alias, + query, + search_request.collectors, + search_request.is_fieldnorms_scoring_enabled, + search_request.load_cache, + search_request.store_cache, + ) + .await?; + Ok(self.index_registry.finalize_extraction(collector_outputs).await?) + } + /// Merge several segments into a single one #[instrument(skip(self, merge_segments_request), fields(index_name = merge_segments_request.index_name))] pub async fn merge_segments(&self, merge_segments_request: proto::MergeSegmentsRequest) -> SummaServerResult> { diff --git a/summa-wasm/Cargo.toml b/summa-wasm/Cargo.toml index 3e166140..662fa637 100644 --- a/summa-wasm/Cargo.toml +++ b/summa-wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "summa-wasm" -version = "0.132.2" +version = "0.133.0" authors = ["Pasha Podolsky "] edition = "2021" license-file = "LICENSE" diff --git a/summa-wasm/build.sh b/summa-wasm/build.sh index 87ef369e..c959c8bd 100755 --- a/summa-wasm/build.sh +++ b/summa-wasm/build.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash npx protoc \ - --ts_out src/grpc-web/ \ - --ts_opt long_type_string \ + --ts_out src/grpc-web \ --ts_opt use_proto_field_name \ --proto_path ../summa-proto/proto \ ../summa-proto/proto/*.proto diff --git a/summa-wasm/crate/web_index_registry.rs b/summa-wasm/crate/web_index_registry.rs index 7cdb5576..b836ed2f 100644 --- a/summa-wasm/crate/web_index_registry.rs +++ b/summa-wasm/crate/web_index_registry.rs @@ -1,5 +1,7 @@ use std::sync::Arc; +use js_sys::Uint8Array; +use prost::Message; use serde::Serialize; use serde_wasm_bindgen::Serializer; use summa_core::components::{IndexHolder, IndexRegistry, SummaDocument}; @@ -48,6 +50,14 @@ impl WrappedIndexRegistry { Ok(self.search_internal(search_request).await.map_err(Error::from)?.serialize(&serializer)?) } + /// Do pooled search + #[wasm_bindgen] + pub async fn search_by_binary_proto(&self, search_request: Uint8Array) -> Result { + let search_request: proto::SearchRequest = proto::SearchRequest::decode(search_request.to_vec().as_slice()).expect("cannot decode proto"); + let serializer = Serializer::new().serialize_maps_as_objects(true).serialize_large_number_types_as_bigints(true); + Ok(self.search_internal(search_request).await.map_err(Error::from)?.serialize(&serializer)?) + } + async fn search_internal(&self, search_request: proto::SearchRequest) -> SummaResult> { info!(action = "search", search_request = ?search_request); let index_holder = self.index_registry.get_index_holder(&search_request.index_alias).await?; diff --git a/summa-wasm/package.json b/summa-wasm/package.json index dad64dc4..6017ed9e 100644 --- a/summa-wasm/package.json +++ b/summa-wasm/package.json @@ -1,7 +1,7 @@ { "name": "summa-wasm", "description": "WASM-bindings for Summa", - "version": "0.132.2", + "version": "0.133.0", "keywords": [ "search", "database", diff --git a/summa-wasm/src/grpc-web/consumer_service.client.ts b/summa-wasm/src/grpc-web/consumer_service.client.ts index bb58ca99..47addad5 100644 --- a/summa-wasm/src/grpc-web/consumer_service.client.ts +++ b/summa-wasm/src/grpc-web/consumer_service.client.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "consumer_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import type { RpcTransport } from "@protobuf-ts/runtime-rpc"; diff --git a/summa-wasm/src/grpc-web/consumer_service.ts b/summa-wasm/src/grpc-web/consumer_service.ts index 72e37b9d..ca05f9a4 100644 --- a/summa-wasm/src/grpc-web/consumer_service.ts +++ b/summa-wasm/src/grpc-web/consumer_service.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "consumer_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import { ServiceType } from "@protobuf-ts/runtime-rpc"; diff --git a/summa-wasm/src/grpc-web/dag_pb.ts b/summa-wasm/src/grpc-web/dag_pb.ts index 482fff36..7a8b4ef5 100644 --- a/summa-wasm/src/grpc-web/dag_pb.ts +++ b/summa-wasm/src/grpc-web/dag_pb.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "dag_pb.proto" (package "dag_pb", syntax proto3) // tslint:disable import type { BinaryWriteOptions } from "@protobuf-ts/runtime"; @@ -32,7 +32,7 @@ export interface PBLink { * * @generated from protobuf field: optional uint64 t_size = 3; */ - t_size?: string; + t_size?: bigint; } /** * @generated from protobuf message dag_pb.PBNode @@ -57,7 +57,7 @@ class PBLink$Type extends MessageType { super("dag_pb.PBLink", [ { no: 1, name: "hash", kind: "scalar", opt: true, T: 12 /*ScalarType.BYTES*/ }, { no: 2, name: "name", kind: "scalar", opt: true, T: 9 /*ScalarType.STRING*/ }, - { no: 3, name: "t_size", kind: "scalar", localName: "t_size", opt: true, T: 4 /*ScalarType.UINT64*/ } + { no: 3, name: "t_size", kind: "scalar", localName: "t_size", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } ]); } create(value?: PartialMessage): PBLink { @@ -79,7 +79,7 @@ class PBLink$Type extends MessageType { message.name = reader.string(); break; case /* optional uint64 t_size */ 3: - message.t_size = reader.uint64().toString(); + message.t_size = reader.uint64().toBigInt(); break; default: let u = options.readUnknownField; diff --git a/summa-wasm/src/grpc-web/index.ts b/summa-wasm/src/grpc-web/index.ts new file mode 100644 index 00000000..47dce7d3 --- /dev/null +++ b/summa-wasm/src/grpc-web/index.ts @@ -0,0 +1,4 @@ +export * as index_service from "./index_service" +export * as query from "./query" +export * as search_service from "./search_service" +export * as search_service_client from "./search_service.client" \ No newline at end of file diff --git a/summa-wasm/src/grpc-web/index_service.client.ts b/summa-wasm/src/grpc-web/index_service.client.ts index afb66f2a..8ad3788f 100644 --- a/summa-wasm/src/grpc-web/index_service.client.ts +++ b/summa-wasm/src/grpc-web/index_service.client.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "index_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import type { RpcTransport } from "@protobuf-ts/runtime-rpc"; diff --git a/summa-wasm/src/grpc-web/index_service.ts b/summa-wasm/src/grpc-web/index_service.ts index 376b4bd7..4c8820d7 100644 --- a/summa-wasm/src/grpc-web/index_service.ts +++ b/summa-wasm/src/grpc-web/index_service.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "index_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import { ServiceType } from "@protobuf-ts/runtime-rpc"; @@ -273,7 +273,7 @@ export interface IndexAttributes { * * @generated from protobuf field: uint64 created_at = 1; */ - created_at: string; + created_at: bigint; /** * Unique fields of the index. Summa maintains unique constraint on them and uses for deduplicating data * @@ -401,7 +401,7 @@ export interface DeleteDocumentsResponse { /** * @generated from protobuf field: uint64 deleted_documents = 1; */ - deleted_documents: string; + deleted_documents: bigint; } /** * @generated from protobuf message summa.proto.DeleteIndexRequest @@ -497,11 +497,11 @@ export interface IndexDocumentStreamResponse { /** * @generated from protobuf field: uint64 success_docs = 2; */ - success_docs: string; + success_docs: bigint; /** * @generated from protobuf field: uint64 failed_docs = 3; */ - failed_docs: string; + failed_docs: bigint; } /** * @generated from protobuf message summa.proto.IndexDocumentRequest @@ -617,7 +617,7 @@ export interface VacuumIndexResponse { /** * @generated from protobuf field: uint64 freed_space_bytes = 1; */ - freed_space_bytes: string; + freed_space_bytes: bigint; } /** * @generated from protobuf message summa.proto.WarmupIndexRequest @@ -674,7 +674,7 @@ export interface CacheConfig { * * @generated from protobuf field: uint64 cache_size = 1; */ - cache_size: string; + cache_size: bigint; } /** * Remote HTTP engine config @@ -737,7 +737,7 @@ export interface TemporalMergePolicy { /** * @generated from protobuf field: uint64 merge_older_then_secs = 1; */ - merge_older_then_secs: string; + merge_older_then_secs: bigint; } /** * Description of the `IndexEngine` responsible for managing files in the persistent storage @@ -805,7 +805,7 @@ export interface IndexDescription { * * @generated from protobuf field: uint64 num_docs = 4; */ - num_docs: string; + num_docs: bigint; /** * Used compression for `store` * @@ -1674,7 +1674,7 @@ export const MappedField = new MappedField$Type(); class IndexAttributes$Type extends MessageType { constructor() { super("summa.proto.IndexAttributes", [ - { no: 1, name: "created_at", kind: "scalar", localName: "created_at", T: 4 /*ScalarType.UINT64*/ }, + { no: 1, name: "created_at", kind: "scalar", localName: "created_at", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, { no: 2, name: "unique_fields", kind: "scalar", localName: "unique_fields", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ }, { no: 4, name: "multi_fields", kind: "scalar", localName: "multi_fields", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ }, { no: 6, name: "description", kind: "scalar", opt: true, T: 9 /*ScalarType.STRING*/ }, @@ -1683,7 +1683,7 @@ class IndexAttributes$Type extends MessageType { ]); } create(value?: PartialMessage): IndexAttributes { - const message = { created_at: "0", unique_fields: [], multi_fields: [], conflict_strategy: 0, mapped_fields: [] }; + const message = { created_at: 0n, unique_fields: [], multi_fields: [], conflict_strategy: 0, mapped_fields: [] }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -1695,7 +1695,7 @@ class IndexAttributes$Type extends MessageType { let [fieldNo, wireType] = reader.tag(); switch (fieldNo) { case /* uint64 created_at */ 1: - message.created_at = reader.uint64().toString(); + message.created_at = reader.uint64().toBigInt(); break; case /* repeated string unique_fields */ 2: message.unique_fields.push(reader.string()); @@ -1725,7 +1725,7 @@ class IndexAttributes$Type extends MessageType { } internalBinaryWrite(message: IndexAttributes, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { /* uint64 created_at = 1; */ - if (message.created_at !== "0") + if (message.created_at !== 0n) writer.tag(1, WireType.Varint).uint64(message.created_at); /* repeated string unique_fields = 2; */ for (let i = 0; i < message.unique_fields.length; i++) @@ -1973,11 +1973,11 @@ export const DeleteDocumentsRequest = new DeleteDocumentsRequest$Type(); class DeleteDocumentsResponse$Type extends MessageType { constructor() { super("summa.proto.DeleteDocumentsResponse", [ - { no: 1, name: "deleted_documents", kind: "scalar", localName: "deleted_documents", T: 4 /*ScalarType.UINT64*/ } + { no: 1, name: "deleted_documents", kind: "scalar", localName: "deleted_documents", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } ]); } create(value?: PartialMessage): DeleteDocumentsResponse { - const message = { deleted_documents: "0" }; + const message = { deleted_documents: 0n }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -1989,7 +1989,7 @@ class DeleteDocumentsResponse$Type extends MessageType let [fieldNo, wireType] = reader.tag(); switch (fieldNo) { case /* uint64 deleted_documents */ 1: - message.deleted_documents = reader.uint64().toString(); + message.deleted_documents = reader.uint64().toBigInt(); break; default: let u = options.readUnknownField; @@ -2004,7 +2004,7 @@ class DeleteDocumentsResponse$Type extends MessageType } internalBinaryWrite(message: DeleteDocumentsResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { /* uint64 deleted_documents = 1; */ - if (message.deleted_documents !== "0") + if (message.deleted_documents !== 0n) writer.tag(1, WireType.Varint).uint64(message.deleted_documents); let u = options.writeUnknownFields; if (u !== false) @@ -2432,12 +2432,12 @@ class IndexDocumentStreamResponse$Type extends MessageType): IndexDocumentStreamResponse { - const message = { elapsed_secs: 0, success_docs: "0", failed_docs: "0" }; + const message = { elapsed_secs: 0, success_docs: 0n, failed_docs: 0n }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -2452,10 +2452,10 @@ class IndexDocumentStreamResponse$Type extends MessageType { constructor() { super("summa.proto.VacuumIndexResponse", [ - { no: 1, name: "freed_space_bytes", kind: "scalar", localName: "freed_space_bytes", T: 4 /*ScalarType.UINT64*/ } + { no: 1, name: "freed_space_bytes", kind: "scalar", localName: "freed_space_bytes", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } ]); } create(value?: PartialMessage): VacuumIndexResponse { - const message = { freed_space_bytes: "0" }; + const message = { freed_space_bytes: 0n }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -2952,7 +2952,7 @@ class VacuumIndexResponse$Type extends MessageType { let [fieldNo, wireType] = reader.tag(); switch (fieldNo) { case /* uint64 freed_space_bytes */ 1: - message.freed_space_bytes = reader.uint64().toString(); + message.freed_space_bytes = reader.uint64().toBigInt(); break; default: let u = options.readUnknownField; @@ -2967,7 +2967,7 @@ class VacuumIndexResponse$Type extends MessageType { } internalBinaryWrite(message: VacuumIndexResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { /* uint64 freed_space_bytes = 1; */ - if (message.freed_space_bytes !== "0") + if (message.freed_space_bytes !== 0n) writer.tag(1, WireType.Varint).uint64(message.freed_space_bytes); let u = options.writeUnknownFields; if (u !== false) @@ -3178,11 +3178,11 @@ export const MemoryEngineConfig = new MemoryEngineConfig$Type(); class CacheConfig$Type extends MessageType { constructor() { super("summa.proto.CacheConfig", [ - { no: 1, name: "cache_size", kind: "scalar", localName: "cache_size", T: 4 /*ScalarType.UINT64*/ } + { no: 1, name: "cache_size", kind: "scalar", localName: "cache_size", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } ]); } create(value?: PartialMessage): CacheConfig { - const message = { cache_size: "0" }; + const message = { cache_size: 0n }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -3194,7 +3194,7 @@ class CacheConfig$Type extends MessageType { let [fieldNo, wireType] = reader.tag(); switch (fieldNo) { case /* uint64 cache_size */ 1: - message.cache_size = reader.uint64().toString(); + message.cache_size = reader.uint64().toBigInt(); break; default: let u = options.readUnknownField; @@ -3209,7 +3209,7 @@ class CacheConfig$Type extends MessageType { } internalBinaryWrite(message: CacheConfig, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { /* uint64 cache_size = 1; */ - if (message.cache_size !== "0") + if (message.cache_size !== 0n) writer.tag(1, WireType.Varint).uint64(message.cache_size); let u = options.writeUnknownFields; if (u !== false) @@ -3363,11 +3363,11 @@ export const LogMergePolicy = new LogMergePolicy$Type(); class TemporalMergePolicy$Type extends MessageType { constructor() { super("summa.proto.TemporalMergePolicy", [ - { no: 1, name: "merge_older_then_secs", kind: "scalar", localName: "merge_older_then_secs", T: 4 /*ScalarType.UINT64*/ } + { no: 1, name: "merge_older_then_secs", kind: "scalar", localName: "merge_older_then_secs", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } ]); } create(value?: PartialMessage): TemporalMergePolicy { - const message = { merge_older_then_secs: "0" }; + const message = { merge_older_then_secs: 0n }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -3379,7 +3379,7 @@ class TemporalMergePolicy$Type extends MessageType { let [fieldNo, wireType] = reader.tag(); switch (fieldNo) { case /* uint64 merge_older_then_secs */ 1: - message.merge_older_then_secs = reader.uint64().toString(); + message.merge_older_then_secs = reader.uint64().toBigInt(); break; default: let u = options.readUnknownField; @@ -3394,7 +3394,7 @@ class TemporalMergePolicy$Type extends MessageType { } internalBinaryWrite(message: TemporalMergePolicy, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { /* uint64 merge_older_then_secs = 1; */ - if (message.merge_older_then_secs !== "0") + if (message.merge_older_then_secs !== 0n) writer.tag(1, WireType.Varint).uint64(message.merge_older_then_secs); let u = options.writeUnknownFields; if (u !== false) @@ -3497,13 +3497,13 @@ class IndexDescription$Type extends MessageType { { no: 1, name: "index_name", kind: "scalar", localName: "index_name", T: 9 /*ScalarType.STRING*/ }, { no: 2, name: "index_aliases", kind: "scalar", localName: "index_aliases", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ }, { no: 3, name: "index_engine", kind: "message", localName: "index_engine", T: () => IndexEngineConfig }, - { no: 4, name: "num_docs", kind: "scalar", localName: "num_docs", T: 4 /*ScalarType.UINT64*/ }, + { no: 4, name: "num_docs", kind: "scalar", localName: "num_docs", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, { no: 5, name: "compression", kind: "enum", T: () => ["summa.proto.Compression", Compression] }, { no: 6, name: "index_attributes", kind: "message", localName: "index_attributes", T: () => IndexAttributes } ]); } create(value?: PartialMessage): IndexDescription { - const message = { index_name: "", index_aliases: [], num_docs: "0", compression: 0 }; + const message = { index_name: "", index_aliases: [], num_docs: 0n, compression: 0 }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -3524,7 +3524,7 @@ class IndexDescription$Type extends MessageType { message.index_engine = IndexEngineConfig.internalBinaryRead(reader, reader.uint32(), options, message.index_engine); break; case /* uint64 num_docs */ 4: - message.num_docs = reader.uint64().toString(); + message.num_docs = reader.uint64().toBigInt(); break; case /* summa.proto.Compression compression */ 5: message.compression = reader.int32(); @@ -3554,7 +3554,7 @@ class IndexDescription$Type extends MessageType { if (message.index_engine) IndexEngineConfig.internalBinaryWrite(message.index_engine, writer.tag(3, WireType.LengthDelimited).fork(), options).join(); /* uint64 num_docs = 4; */ - if (message.num_docs !== "0") + if (message.num_docs !== 0n) writer.tag(4, WireType.Varint).uint64(message.num_docs); /* summa.proto.Compression compression = 5; */ if (message.compression !== 0) diff --git a/summa-wasm/src/grpc-web/public_service.client.ts b/summa-wasm/src/grpc-web/public_service.client.ts new file mode 100644 index 00000000..a365090f --- /dev/null +++ b/summa-wasm/src/grpc-web/public_service.client.ts @@ -0,0 +1,45 @@ +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name +// @generated from protobuf file "public_service.proto" (package "summa.proto", syntax proto3) +// tslint:disable +import type { RpcTransport } from "@protobuf-ts/runtime-rpc"; +import type { ServiceInfo } from "@protobuf-ts/runtime-rpc"; +import { PublicApi } from "./public_service"; +import { stackIntercept } from "@protobuf-ts/runtime-rpc"; +import type { SearchResponse } from "./query"; +import type { SearchRequest } from "./search_service"; +import type { UnaryCall } from "@protobuf-ts/runtime-rpc"; +import type { RpcOptions } from "@protobuf-ts/runtime-rpc"; +/** + * Searches documents in the stored indices + * + * @generated from protobuf service summa.proto.PublicApi + */ +export interface IPublicApiClient { + /** + * Make search in Summa + * + * @generated from protobuf rpc: search(summa.proto.SearchRequest) returns (summa.proto.SearchResponse); + */ + search(input: SearchRequest, options?: RpcOptions): UnaryCall; +} +/** + * Searches documents in the stored indices + * + * @generated from protobuf service summa.proto.PublicApi + */ +export class PublicApiClient implements IPublicApiClient, ServiceInfo { + typeName = PublicApi.typeName; + methods = PublicApi.methods; + options = PublicApi.options; + constructor(private readonly _transport: RpcTransport) { + } + /** + * Make search in Summa + * + * @generated from protobuf rpc: search(summa.proto.SearchRequest) returns (summa.proto.SearchResponse); + */ + search(input: SearchRequest, options?: RpcOptions): UnaryCall { + const method = this.methods[0], opt = this._transport.mergeOptions(options); + return stackIntercept("unary", this._transport, method, opt, input); + } +} diff --git a/summa-wasm/src/grpc-web/public_service.ts b/summa-wasm/src/grpc-web/public_service.ts new file mode 100644 index 00000000..d7ae0157 --- /dev/null +++ b/summa-wasm/src/grpc-web/public_service.ts @@ -0,0 +1,12 @@ +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name +// @generated from protobuf file "public_service.proto" (package "summa.proto", syntax proto3) +// tslint:disable +import { SearchResponse } from "./query"; +import { SearchRequest } from "./search_service"; +import { ServiceType } from "@protobuf-ts/runtime-rpc"; +/** + * @generated ServiceType for protobuf service summa.proto.PublicApi + */ +export const PublicApi = new ServiceType("summa.proto.PublicApi", [ + { name: "search", options: {}, I: SearchRequest, O: SearchResponse } +]); diff --git a/summa-wasm/src/grpc-web/query.ts b/summa-wasm/src/grpc-web/query.ts index 74b63f58..24e019c6 100644 --- a/summa-wasm/src/grpc-web/query.ts +++ b/summa-wasm/src/grpc-web/query.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "query.proto" (package "summa.proto", syntax proto3) // tslint:disable import type { BinaryWriteOptions } from "@protobuf-ts/runtime"; @@ -289,27 +289,27 @@ export interface MoreLikeThisQuery { /** * @generated from protobuf field: optional uint64 min_doc_frequency = 2; */ - min_doc_frequency?: string; + min_doc_frequency?: bigint; /** * @generated from protobuf field: optional uint64 max_doc_frequency = 3; */ - max_doc_frequency?: string; + max_doc_frequency?: bigint; /** * @generated from protobuf field: optional uint64 min_term_frequency = 4; */ - min_term_frequency?: string; + min_term_frequency?: bigint; /** * @generated from protobuf field: optional uint64 max_query_terms = 5; */ - max_query_terms?: string; + max_query_terms?: bigint; /** * @generated from protobuf field: optional uint64 min_word_length = 6; */ - min_word_length?: string; + min_word_length?: bigint; /** * @generated from protobuf field: optional uint64 max_word_length = 7; */ - max_word_length?: string; + max_word_length?: bigint; /** * @generated from protobuf field: optional string boost = 8; */ @@ -458,7 +458,7 @@ export interface Score { /** * @generated from protobuf field: uint64 u64_score = 2; */ - u64_score: string; + u64_score: bigint; } | { oneofKind: undefined; }; @@ -656,7 +656,7 @@ export interface FacetCollectorOutput { * @generated from protobuf field: map facet_counts = 1; */ facet_counts: { - [key: string]: string; + [key: string]: bigint; }; } /** @@ -671,6 +671,10 @@ export interface ReservoirSamplingCollector { * @generated from protobuf field: repeated string fields = 2; */ fields: string[]; + /** + * @generated from protobuf field: repeated string removed_fields = 3; + */ + removed_fields: string[]; } /** * @generated from protobuf message summa.proto.RandomDocument @@ -728,6 +732,10 @@ export interface TopDocsCollector { * @generated from protobuf field: repeated string fields = 6; */ fields: string[]; + /** + * @generated from protobuf field: repeated string removed_fields = 7; + */ + removed_fields: string[]; } /** * @generated from protobuf message summa.proto.DocumentsCollectorOutput @@ -1633,12 +1641,12 @@ class MoreLikeThisQuery$Type extends MessageType { constructor() { super("summa.proto.MoreLikeThisQuery", [ { no: 1, name: "document", kind: "scalar", T: 9 /*ScalarType.STRING*/ }, - { no: 2, name: "min_doc_frequency", kind: "scalar", localName: "min_doc_frequency", opt: true, T: 4 /*ScalarType.UINT64*/ }, - { no: 3, name: "max_doc_frequency", kind: "scalar", localName: "max_doc_frequency", opt: true, T: 4 /*ScalarType.UINT64*/ }, - { no: 4, name: "min_term_frequency", kind: "scalar", localName: "min_term_frequency", opt: true, T: 4 /*ScalarType.UINT64*/ }, - { no: 5, name: "max_query_terms", kind: "scalar", localName: "max_query_terms", opt: true, T: 4 /*ScalarType.UINT64*/ }, - { no: 6, name: "min_word_length", kind: "scalar", localName: "min_word_length", opt: true, T: 4 /*ScalarType.UINT64*/ }, - { no: 7, name: "max_word_length", kind: "scalar", localName: "max_word_length", opt: true, T: 4 /*ScalarType.UINT64*/ }, + { no: 2, name: "min_doc_frequency", kind: "scalar", localName: "min_doc_frequency", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 3, name: "max_doc_frequency", kind: "scalar", localName: "max_doc_frequency", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 4, name: "min_term_frequency", kind: "scalar", localName: "min_term_frequency", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 5, name: "max_query_terms", kind: "scalar", localName: "max_query_terms", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 6, name: "min_word_length", kind: "scalar", localName: "min_word_length", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 7, name: "max_word_length", kind: "scalar", localName: "max_word_length", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, { no: 8, name: "boost", kind: "scalar", opt: true, T: 9 /*ScalarType.STRING*/ }, { no: 9, name: "stop_words", kind: "scalar", localName: "stop_words", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ } ]); @@ -1659,22 +1667,22 @@ class MoreLikeThisQuery$Type extends MessageType { message.document = reader.string(); break; case /* optional uint64 min_doc_frequency */ 2: - message.min_doc_frequency = reader.uint64().toString(); + message.min_doc_frequency = reader.uint64().toBigInt(); break; case /* optional uint64 max_doc_frequency */ 3: - message.max_doc_frequency = reader.uint64().toString(); + message.max_doc_frequency = reader.uint64().toBigInt(); break; case /* optional uint64 min_term_frequency */ 4: - message.min_term_frequency = reader.uint64().toString(); + message.min_term_frequency = reader.uint64().toBigInt(); break; case /* optional uint64 max_query_terms */ 5: - message.max_query_terms = reader.uint64().toString(); + message.max_query_terms = reader.uint64().toBigInt(); break; case /* optional uint64 min_word_length */ 6: - message.min_word_length = reader.uint64().toString(); + message.min_word_length = reader.uint64().toBigInt(); break; case /* optional uint64 max_word_length */ 7: - message.max_word_length = reader.uint64().toString(); + message.max_word_length = reader.uint64().toBigInt(); break; case /* optional string boost */ 8: message.boost = reader.string(); @@ -2229,7 +2237,7 @@ class Score$Type extends MessageType { constructor() { super("summa.proto.Score", [ { no: 1, name: "f64_score", kind: "scalar", localName: "f64_score", oneof: "score", T: 1 /*ScalarType.DOUBLE*/ }, - { no: 2, name: "u64_score", kind: "scalar", localName: "u64_score", oneof: "score", T: 4 /*ScalarType.UINT64*/ } + { no: 2, name: "u64_score", kind: "scalar", localName: "u64_score", oneof: "score", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } ]); } create(value?: PartialMessage): Score { @@ -2253,7 +2261,7 @@ class Score$Type extends MessageType { case /* uint64 u64_score */ 2: message.score = { oneofKind: "u64_score", - u64_score: reader.uint64().toString() + u64_score: reader.uint64().toBigInt() }; break; default: @@ -2855,7 +2863,7 @@ export const FacetCollector = new FacetCollector$Type(); class FacetCollectorOutput$Type extends MessageType { constructor() { super("summa.proto.FacetCollectorOutput", [ - { no: 1, name: "facet_counts", kind: "map", localName: "facet_counts", K: 9 /*ScalarType.STRING*/, V: { kind: "scalar", T: 4 /*ScalarType.UINT64*/ } } + { no: 1, name: "facet_counts", kind: "map", localName: "facet_counts", K: 9 /*ScalarType.STRING*/, V: { kind: "scalar", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } } ]); } create(value?: PartialMessage): FacetCollectorOutput { @@ -2893,12 +2901,12 @@ class FacetCollectorOutput$Type extends MessageType { key = reader.string(); break; case 2: - val = reader.uint64().toString(); + val = reader.uint64().toBigInt(); break; default: throw new globalThis.Error("unknown map entry field for field summa.proto.FacetCollectorOutput.facet_counts"); } } - map[key ?? ""] = val ?? "0"; + map[key ?? ""] = val ?? 0n; } internalBinaryWrite(message: FacetCollectorOutput, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { /* map facet_counts = 1; */ @@ -2919,11 +2927,12 @@ class ReservoirSamplingCollector$Type extends MessageType): ReservoirSamplingCollector { - const message = { limit: 0, fields: [] }; + const message = { limit: 0, fields: [], removed_fields: [] }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -2940,6 +2949,9 @@ class ReservoirSamplingCollector$Type extends MessageType { { no: 3, name: "scorer", kind: "message", T: () => Scorer }, { no: 4, name: "snippet_configs", kind: "map", localName: "snippet_configs", K: 9 /*ScalarType.STRING*/, V: { kind: "scalar", T: 13 /*ScalarType.UINT32*/ } }, { no: 5, name: "explain", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }, - { no: 6, name: "fields", kind: "scalar", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ } + { no: 6, name: "fields", kind: "scalar", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ }, + { no: 7, name: "removed_fields", kind: "scalar", localName: "removed_fields", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ } ]); } create(value?: PartialMessage): TopDocsCollector { - const message = { limit: 0, offset: 0, snippet_configs: {}, explain: false, fields: [] }; + const message = { limit: 0, offset: 0, snippet_configs: {}, explain: false, fields: [], removed_fields: [] }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) reflectionMergePartial(this, message, value); @@ -3118,6 +3134,9 @@ class TopDocsCollector$Type extends MessageType { case /* repeated string fields */ 6: message.fields.push(reader.string()); break; + case /* repeated string removed_fields */ 7: + message.removed_fields.push(reader.string()); + break; default: let u = options.readUnknownField; if (u === "throw") @@ -3164,6 +3183,9 @@ class TopDocsCollector$Type extends MessageType { /* repeated string fields = 6; */ for (let i = 0; i < message.fields.length; i++) writer.tag(6, WireType.LengthDelimited).string(message.fields[i]); + /* repeated string removed_fields = 7; */ + for (let i = 0; i < message.removed_fields.length; i++) + writer.tag(7, WireType.LengthDelimited).string(message.removed_fields[i]); let u = options.writeUnknownFields; if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); diff --git a/summa-wasm/src/grpc-web/reflection_service.client.ts b/summa-wasm/src/grpc-web/reflection_service.client.ts index cbf78219..dc590389 100644 --- a/summa-wasm/src/grpc-web/reflection_service.client.ts +++ b/summa-wasm/src/grpc-web/reflection_service.client.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "reflection_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import type { RpcTransport } from "@protobuf-ts/runtime-rpc"; diff --git a/summa-wasm/src/grpc-web/reflection_service.ts b/summa-wasm/src/grpc-web/reflection_service.ts index f803df8a..af4fb984 100644 --- a/summa-wasm/src/grpc-web/reflection_service.ts +++ b/summa-wasm/src/grpc-web/reflection_service.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "reflection_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import { ServiceType } from "@protobuf-ts/runtime-rpc"; diff --git a/summa-wasm/src/grpc-web/search_service.client.ts b/summa-wasm/src/grpc-web/search_service.client.ts index 5799d13f..86f0fefb 100644 --- a/summa-wasm/src/grpc-web/search_service.client.ts +++ b/summa-wasm/src/grpc-web/search_service.client.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "search_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import type { RpcTransport } from "@protobuf-ts/runtime-rpc"; diff --git a/summa-wasm/src/grpc-web/search_service.ts b/summa-wasm/src/grpc-web/search_service.ts index 524a4418..c8bd3848 100644 --- a/summa-wasm/src/grpc-web/search_service.ts +++ b/summa-wasm/src/grpc-web/search_service.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "search_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import { SearchResponse } from "./query"; diff --git a/summa-wasm/src/grpc-web/unixfs.ts b/summa-wasm/src/grpc-web/unixfs.ts index ba552c49..88f83854 100644 --- a/summa-wasm/src/grpc-web/unixfs.ts +++ b/summa-wasm/src/grpc-web/unixfs.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "unixfs.proto" (package "unixfs", syntax proto3) // tslint:disable import type { BinaryWriteOptions } from "@protobuf-ts/runtime"; @@ -26,19 +26,19 @@ export interface Data { /** * @generated from protobuf field: optional uint64 filesize = 3; */ - filesize?: string; + filesize?: bigint; /** * @generated from protobuf field: repeated uint64 blocksizes = 4; */ - blocksizes: string[]; + blocksizes: bigint[]; /** * @generated from protobuf field: optional uint64 hashType = 5; */ - hashType?: string; + hashType?: bigint; /** * @generated from protobuf field: optional uint64 fanout = 6; */ - fanout?: string; + fanout?: bigint; } /** * @generated from protobuf enum unixfs.Data.DataType @@ -84,10 +84,10 @@ class Data$Type extends MessageType { super("unixfs.Data", [ { no: 1, name: "type", kind: "enum", T: () => ["unixfs.Data.DataType", Data_DataType] }, { no: 2, name: "data", kind: "scalar", opt: true, T: 12 /*ScalarType.BYTES*/ }, - { no: 3, name: "filesize", kind: "scalar", opt: true, T: 4 /*ScalarType.UINT64*/ }, - { no: 4, name: "blocksizes", kind: "scalar", repeat: 1 /*RepeatType.PACKED*/, T: 4 /*ScalarType.UINT64*/ }, - { no: 5, name: "hashType", kind: "scalar", opt: true, T: 4 /*ScalarType.UINT64*/ }, - { no: 6, name: "fanout", kind: "scalar", opt: true, T: 4 /*ScalarType.UINT64*/ } + { no: 3, name: "filesize", kind: "scalar", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 4, name: "blocksizes", kind: "scalar", repeat: 1 /*RepeatType.PACKED*/, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 5, name: "hashType", kind: "scalar", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }, + { no: 6, name: "fanout", kind: "scalar", opt: true, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ } ]); } create(value?: PartialMessage): Data { @@ -109,20 +109,20 @@ class Data$Type extends MessageType { message.data = reader.bytes(); break; case /* optional uint64 filesize */ 3: - message.filesize = reader.uint64().toString(); + message.filesize = reader.uint64().toBigInt(); break; case /* repeated uint64 blocksizes */ 4: if (wireType === WireType.LengthDelimited) for (let e = reader.int32() + reader.pos; reader.pos < e;) - message.blocksizes.push(reader.uint64().toString()); + message.blocksizes.push(reader.uint64().toBigInt()); else - message.blocksizes.push(reader.uint64().toString()); + message.blocksizes.push(reader.uint64().toBigInt()); break; case /* optional uint64 hashType */ 5: - message.hashType = reader.uint64().toString(); + message.hashType = reader.uint64().toBigInt(); break; case /* optional uint64 fanout */ 6: - message.fanout = reader.uint64().toString(); + message.fanout = reader.uint64().toBigInt(); break; default: let u = options.readUnknownField; diff --git a/summa-wasm/src/grpc-web/utils.ts b/summa-wasm/src/grpc-web/utils.ts index 14b753a1..f9382b79 100644 --- a/summa-wasm/src/grpc-web/utils.ts +++ b/summa-wasm/src/grpc-web/utils.ts @@ -1,4 +1,4 @@ -// @generated by protobuf-ts 2.9.1 with parameter long_type_string,use_proto_field_name +// @generated by protobuf-ts 2.9.1 with parameter use_proto_field_name // @generated from protobuf file "utils.proto" (package "summa.proto", syntax proto3) // tslint:disable import type { BinaryWriteOptions } from "@protobuf-ts/runtime"; diff --git a/summa-wasm/src/index-registry.ts b/summa-wasm/src/index-registry.ts index dffc18f3..de4de4e0 100644 --- a/summa-wasm/src/index-registry.ts +++ b/summa-wasm/src/index-registry.ts @@ -6,6 +6,7 @@ export interface IIndexRegistry { add(index_name: string, index_engine_config: IndexEngineConfig): Promise; delete(index_name: string): Promise; search(search_request: SearchRequest): Promise; + search_by_binary_proto(search_request_proto_bytes: Uint8Array): Promise; warmup(index_name: string): Promise; index_document(index_name: string, document: string): Promise; commit(index_name: string): Promise; @@ -57,6 +58,9 @@ export class IndexRegistry implements IIndexRegistry { async search(search_request: SearchRequest): Promise { return await this.registry!.search(search_request); } + async search_by_binary_proto(search_request_bytes_proto: Uint8Array): Promise { + return await this.registry!.search_by_binary_proto(search_request_bytes_proto); + } async warmup(index_name: string) { return await this.registry!.warmup(index_name); } diff --git a/summa-wasm/src/index.ts b/summa-wasm/src/index.ts index c491714b..adcaec4e 100644 --- a/summa-wasm/src/index.ts +++ b/summa-wasm/src/index.ts @@ -8,3 +8,4 @@ export { } from "./remote-index-registry"; export * as seeds from "./seeds"; export * as utils from "./utils"; +export * as grpc_web from "./grpc-web" \ No newline at end of file diff --git a/summa-wasm/src/remote-index-registry.ts b/summa-wasm/src/remote-index-registry.ts index cfcf4d5e..1626d456 100644 --- a/summa-wasm/src/remote-index-registry.ts +++ b/summa-wasm/src/remote-index-registry.ts @@ -29,6 +29,10 @@ export class RemoteIndexRegistry implements IIndexRegistry { return this.index_registry.search(search_request) } + search_by_binary_proto(search_request_proto_bytes: Uint8Array): Promise { + return this.index_registry.search_by_binary_proto(search_request_proto_bytes) + } + warmup(index_name: string): Promise { return this.index_registry.warmup(index_name); } diff --git a/summa-wasm/src/service-worker.ts b/summa-wasm/src/service-worker.ts index 9177337b..cd2d6b0a 100644 --- a/summa-wasm/src/service-worker.ts +++ b/summa-wasm/src/service-worker.ts @@ -66,6 +66,11 @@ async function handle_request(event: FetchEvent) { let filename = request.url; let url = request.url; let is_development = (new URL(request.url)).host == "localhost:5173" + let is_api_request = request.url.endsWith('/summa.proto.SearchApi/search'); + + if (is_api_request) { + return fetch(event.request) + } let is_immutable_file = filename.endsWith(".fast") || filename.endsWith(".term") || diff --git a/summa-wasm/src/utils.ts b/summa-wasm/src/utils.ts index d36d908c..4f8eea79 100644 --- a/summa-wasm/src/utils.ts +++ b/summa-wasm/src/utils.ts @@ -37,4 +37,4 @@ export function get_ipfs_url(hostname?: string) { ipfs_url = `${ipfs_url}:${window.location.port}`; } return ipfs_url; -} +} \ No newline at end of file