diff --git a/summa-core/src/validators.rs b/summa-core/src/validators.rs index d6448e14..e81760c5 100644 --- a/summa-core/src/validators.rs +++ b/summa-core/src/validators.rs @@ -15,10 +15,16 @@ pub fn parse_fields<'a>(schema: &'a Schema, fields: &'a [String], removed_fields } else if fields.is_empty() { Ok(schema .fields() - .map(|(_, field_entry)| { - schema - .find_field(field_entry.name()) - .ok_or_else(|| ValidationError::MissingField(field_entry.name().to_string())) + .filter_map(|(_, field_entry)| { + if removed_fields.iter().any(|e| e == field_entry.name()) { + None + } else { + Some( + schema + .find_field(field_entry.name()) + .ok_or_else(|| ValidationError::MissingField(field_entry.name().to_string())), + ) + } }) .collect::>()?) } else { diff --git a/summa-proto/proto/public_service.proto b/summa-proto/proto/public_service.proto index e6d234a5..ec6903e5 100644 --- a/summa-proto/proto/public_service.proto +++ b/summa-proto/proto/public_service.proto @@ -1,7 +1,6 @@ syntax = "proto3"; package summa.proto; -import "search_service.proto"; import "query.proto"; diff --git a/summa-proto/proto/query.proto b/summa-proto/proto/query.proto index af94fae4..d38deda6 100644 --- a/summa-proto/proto/query.proto +++ b/summa-proto/proto/query.proto @@ -40,6 +40,20 @@ message QueryParserConfig { optional string query_language = 11; } +message SearchRequest { + // The index name or alias + string index_alias = 1; + // Query DSL. Use `MatchQuery` to pass a free-form query + Query query = 2; + // Every collector is responsible of processing and storing documents and/or their derivatives (like counters) + // to return them to the caller + repeated Collector collectors = 3; + // Is requiring fieldnorms needed for the query? + optional bool is_fieldnorms_scoring_enabled = 4; + optional bool load_cache = 5; + optional bool store_cache = 6; +} + message SearchResponse { // Time spent inside of `search` handler double elapsed_secs = 1; diff --git a/summa-proto/proto/search_service.proto b/summa-proto/proto/search_service.proto index fe908181..322bd143 100644 --- a/summa-proto/proto/search_service.proto +++ b/summa-proto/proto/search_service.proto @@ -9,17 +9,3 @@ service SearchApi { // Make search in Summa rpc search (SearchRequest) returns (SearchResponse) {} } - -message SearchRequest { - // The index name or alias - string index_alias = 1; - // Query DSL. Use `MatchQuery` to pass a free-form query - Query query = 2; - // Every collector is responsible of processing and storing documents and/or their derivatives (like counters) - // to return them to the caller - repeated Collector collectors = 3; - // Is requiring fieldnorms needed for the query? - optional bool is_fieldnorms_scoring_enabled = 4; - optional bool load_cache = 5; - optional bool store_cache = 6; -} diff --git a/summa-server/src/errors.rs b/summa-server/src/errors.rs index 8a470151..e36e6b73 100644 --- a/summa-server/src/errors.rs +++ b/summa-server/src/errors.rs @@ -49,6 +49,8 @@ pub enum Error { Json(#[from] serde_json::Error), #[error("lock_error: {0}")] Lock(#[from] tokio::sync::TryLockError), + #[error("not_allowed_error")] + NotAllowed, #[error("tantivy_error: {0}")] Tantivy(#[from] tantivy::TantivyError), #[error("timeout_error: {0}")] @@ -94,6 +96,7 @@ impl From for tonic::Status { }, Error::Validation(ValidationError::MissingIndex(_)) => tonic::Code::NotFound, Error::Validation(_) => tonic::Code::InvalidArgument, + Error::NotAllowed => tonic::Code::PermissionDenied, Error::Lock(_) => tonic::Code::FailedPrecondition, _ => tonic::Code::Internal, }, diff --git a/summa-server/src/services/api.rs b/summa-server/src/services/api.rs index 4ff90589..8c2833d3 100644 --- a/summa-server/src/services/api.rs +++ b/summa-server/src/services/api.rs @@ -135,6 +135,7 @@ impl Api { .add_service(grpc_reflection_service) .add_service(consumer_service.clone()) .add_service(index_service.clone()) + .add_service(public_service.clone()) .add_service(reflection_service.clone()) .add_service(search_service.clone()); let grpc_listener = Api::set_listener(&api_config.grpc_endpoint)?; diff --git a/summa-server/src/services/index.rs b/summa-server/src/services/index.rs index c0b4ba9d..2465a86a 100644 --- a/summa-server/src/services/index.rs +++ b/summa-server/src/services/index.rs @@ -655,6 +655,9 @@ impl Index { .and_then(|query| query.query) .unwrap_or_else(|| proto::query::Query::All(proto::AllQuery {})); + if search_request.collectors.len() > 2 { + return Err(crate::errors::Error::NotAllowed); + } for collector in &mut search_request.collectors { match &mut collector.collector { Some(proto::collector::Collector::TopDocs(top_docs)) => { @@ -667,7 +670,7 @@ impl Index { reservoir_sampling.removed_fields = vec!["content".to_string()]; } Some(proto::collector::Collector::Count(_)) => {} - _ => panic!("Not allowed"), + _ => return Err(crate::errors::Error::NotAllowed), } } @@ -676,9 +679,9 @@ impl Index { &search_request.index_alias, query, search_request.collectors, - search_request.is_fieldnorms_scoring_enabled, - search_request.load_cache, - search_request.store_cache, + Some(true), + Some(true), + Some(true), ) .await?; Ok(self.index_registry.finalize_extraction(collector_outputs).await?) diff --git a/summa-wasm/Cargo.toml b/summa-wasm/Cargo.toml index 662fa637..5c4f2527 100644 --- a/summa-wasm/Cargo.toml +++ b/summa-wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "summa-wasm" -version = "0.133.0" +version = "0.133.3" authors = ["Pasha Podolsky "] edition = "2021" license-file = "LICENSE" diff --git a/summa-wasm/package.json b/summa-wasm/package.json index 6017ed9e..756a4ca9 100644 --- a/summa-wasm/package.json +++ b/summa-wasm/package.json @@ -1,7 +1,7 @@ { "name": "summa-wasm", "description": "WASM-bindings for Summa", - "version": "0.133.0", + "version": "0.133.3", "keywords": [ "search", "database", diff --git a/summa-wasm/src/grpc-web/index.ts b/summa-wasm/src/grpc-web/index.ts index 47dce7d3..de942cf6 100644 --- a/summa-wasm/src/grpc-web/index.ts +++ b/summa-wasm/src/grpc-web/index.ts @@ -1,4 +1,6 @@ export * as index_service from "./index_service" +export * as public_service from "./public_service" +export * as public_service_client from "./public_service.client" export * as query from "./query" export * as search_service from "./search_service" export * as search_service_client from "./search_service.client" \ No newline at end of file diff --git a/summa-wasm/src/grpc-web/public_service.client.ts b/summa-wasm/src/grpc-web/public_service.client.ts index a365090f..e304de48 100644 --- a/summa-wasm/src/grpc-web/public_service.client.ts +++ b/summa-wasm/src/grpc-web/public_service.client.ts @@ -6,7 +6,7 @@ import type { ServiceInfo } from "@protobuf-ts/runtime-rpc"; import { PublicApi } from "./public_service"; import { stackIntercept } from "@protobuf-ts/runtime-rpc"; import type { SearchResponse } from "./query"; -import type { SearchRequest } from "./search_service"; +import type { SearchRequest } from "./query"; import type { UnaryCall } from "@protobuf-ts/runtime-rpc"; import type { RpcOptions } from "@protobuf-ts/runtime-rpc"; /** diff --git a/summa-wasm/src/grpc-web/public_service.ts b/summa-wasm/src/grpc-web/public_service.ts index d7ae0157..15097802 100644 --- a/summa-wasm/src/grpc-web/public_service.ts +++ b/summa-wasm/src/grpc-web/public_service.ts @@ -2,7 +2,7 @@ // @generated from protobuf file "public_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import { SearchResponse } from "./query"; -import { SearchRequest } from "./search_service"; +import { SearchRequest } from "./query"; import { ServiceType } from "@protobuf-ts/runtime-rpc"; /** * @generated ServiceType for protobuf service summa.proto.PublicApi diff --git a/summa-wasm/src/grpc-web/query.ts b/summa-wasm/src/grpc-web/query.ts index 24e019c6..e84a895d 100644 --- a/summa-wasm/src/grpc-web/query.ts +++ b/summa-wasm/src/grpc-web/query.ts @@ -140,6 +140,44 @@ export interface QueryParserConfig { */ query_language?: string; } +/** + * @generated from protobuf message summa.proto.SearchRequest + */ +export interface SearchRequest { + /** + * The index name or alias + * + * @generated from protobuf field: string index_alias = 1; + */ + index_alias: string; + /** + * Query DSL. Use `MatchQuery` to pass a free-form query + * + * @generated from protobuf field: summa.proto.Query query = 2; + */ + query?: Query; + /** + * Every collector is responsible of processing and storing documents and/or their derivatives (like counters) + * to return them to the caller + * + * @generated from protobuf field: repeated summa.proto.Collector collectors = 3; + */ + collectors: Collector[]; + /** + * Is requiring fieldnorms needed for the query? + * + * @generated from protobuf field: optional bool is_fieldnorms_scoring_enabled = 4; + */ + is_fieldnorms_scoring_enabled?: boolean; + /** + * @generated from protobuf field: optional bool load_cache = 5; + */ + load_cache?: boolean; + /** + * @generated from protobuf field: optional bool store_cache = 6; + */ + store_cache?: boolean; +} /** * @generated from protobuf message summa.proto.SearchResponse */ @@ -1263,6 +1301,88 @@ class QueryParserConfig$Type extends MessageType { */ export const QueryParserConfig = new QueryParserConfig$Type(); // @generated message type with reflection information, may provide speed optimized methods +class SearchRequest$Type extends MessageType { + constructor() { + super("summa.proto.SearchRequest", [ + { no: 1, name: "index_alias", kind: "scalar", localName: "index_alias", T: 9 /*ScalarType.STRING*/ }, + { no: 2, name: "query", kind: "message", T: () => Query }, + { no: 3, name: "collectors", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => Collector }, + { no: 4, name: "is_fieldnorms_scoring_enabled", kind: "scalar", localName: "is_fieldnorms_scoring_enabled", opt: true, T: 8 /*ScalarType.BOOL*/ }, + { no: 5, name: "load_cache", kind: "scalar", localName: "load_cache", opt: true, T: 8 /*ScalarType.BOOL*/ }, + { no: 6, name: "store_cache", kind: "scalar", localName: "store_cache", opt: true, T: 8 /*ScalarType.BOOL*/ } + ]); + } + create(value?: PartialMessage): SearchRequest { + const message = { index_alias: "", collectors: [] }; + globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); + if (value !== undefined) + reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: SearchRequest): SearchRequest { + let message = target ?? this.create(), end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string index_alias */ 1: + message.index_alias = reader.string(); + break; + case /* summa.proto.Query query */ 2: + message.query = Query.internalBinaryRead(reader, reader.uint32(), options, message.query); + break; + case /* repeated summa.proto.Collector collectors */ 3: + message.collectors.push(Collector.internalBinaryRead(reader, reader.uint32(), options)); + break; + case /* optional bool is_fieldnorms_scoring_enabled */ 4: + message.is_fieldnorms_scoring_enabled = reader.bool(); + break; + case /* optional bool load_cache */ 5: + message.load_cache = reader.bool(); + break; + case /* optional bool store_cache */ 6: + message.store_cache = reader.bool(); + break; + default: + let u = options.readUnknownField; + if (u === "throw") + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) + (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: SearchRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string index_alias = 1; */ + if (message.index_alias !== "") + writer.tag(1, WireType.LengthDelimited).string(message.index_alias); + /* summa.proto.Query query = 2; */ + if (message.query) + Query.internalBinaryWrite(message.query, writer.tag(2, WireType.LengthDelimited).fork(), options).join(); + /* repeated summa.proto.Collector collectors = 3; */ + for (let i = 0; i < message.collectors.length; i++) + Collector.internalBinaryWrite(message.collectors[i], writer.tag(3, WireType.LengthDelimited).fork(), options).join(); + /* optional bool is_fieldnorms_scoring_enabled = 4; */ + if (message.is_fieldnorms_scoring_enabled !== undefined) + writer.tag(4, WireType.Varint).bool(message.is_fieldnorms_scoring_enabled); + /* optional bool load_cache = 5; */ + if (message.load_cache !== undefined) + writer.tag(5, WireType.Varint).bool(message.load_cache); + /* optional bool store_cache = 6; */ + if (message.store_cache !== undefined) + writer.tag(6, WireType.Varint).bool(message.store_cache); + let u = options.writeUnknownFields; + if (u !== false) + (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message summa.proto.SearchRequest + */ +export const SearchRequest = new SearchRequest$Type(); +// @generated message type with reflection information, may provide speed optimized methods class SearchResponse$Type extends MessageType { constructor() { super("summa.proto.SearchResponse", [ diff --git a/summa-wasm/src/grpc-web/search_service.client.ts b/summa-wasm/src/grpc-web/search_service.client.ts index 86f0fefb..601fb607 100644 --- a/summa-wasm/src/grpc-web/search_service.client.ts +++ b/summa-wasm/src/grpc-web/search_service.client.ts @@ -6,7 +6,7 @@ import type { ServiceInfo } from "@protobuf-ts/runtime-rpc"; import { SearchApi } from "./search_service"; import { stackIntercept } from "@protobuf-ts/runtime-rpc"; import type { SearchResponse } from "./query"; -import type { SearchRequest } from "./search_service"; +import type { SearchRequest } from "./query"; import type { UnaryCall } from "@protobuf-ts/runtime-rpc"; import type { RpcOptions } from "@protobuf-ts/runtime-rpc"; /** diff --git a/summa-wasm/src/grpc-web/search_service.ts b/summa-wasm/src/grpc-web/search_service.ts index c8bd3848..288e8e70 100644 --- a/summa-wasm/src/grpc-web/search_service.ts +++ b/summa-wasm/src/grpc-web/search_service.ts @@ -2,139 +2,8 @@ // @generated from protobuf file "search_service.proto" (package "summa.proto", syntax proto3) // tslint:disable import { SearchResponse } from "./query"; +import { SearchRequest } from "./query"; import { ServiceType } from "@protobuf-ts/runtime-rpc"; -import type { BinaryWriteOptions } from "@protobuf-ts/runtime"; -import type { IBinaryWriter } from "@protobuf-ts/runtime"; -import { WireType } from "@protobuf-ts/runtime"; -import type { BinaryReadOptions } from "@protobuf-ts/runtime"; -import type { IBinaryReader } from "@protobuf-ts/runtime"; -import { UnknownFieldHandler } from "@protobuf-ts/runtime"; -import type { PartialMessage } from "@protobuf-ts/runtime"; -import { reflectionMergePartial } from "@protobuf-ts/runtime"; -import { MESSAGE_TYPE } from "@protobuf-ts/runtime"; -import { MessageType } from "@protobuf-ts/runtime"; -import { Collector } from "./query"; -import { Query } from "./query"; -/** - * @generated from protobuf message summa.proto.SearchRequest - */ -export interface SearchRequest { - /** - * The index name or alias - * - * @generated from protobuf field: string index_alias = 1; - */ - index_alias: string; - /** - * Query DSL. Use `MatchQuery` to pass a free-form query - * - * @generated from protobuf field: summa.proto.Query query = 2; - */ - query?: Query; - /** - * Every collector is responsible of processing and storing documents and/or their derivatives (like counters) - * to return them to the caller - * - * @generated from protobuf field: repeated summa.proto.Collector collectors = 3; - */ - collectors: Collector[]; - /** - * Is requiring fieldnorms needed for the query? - * - * @generated from protobuf field: optional bool is_fieldnorms_scoring_enabled = 4; - */ - is_fieldnorms_scoring_enabled?: boolean; - /** - * @generated from protobuf field: optional bool load_cache = 5; - */ - load_cache?: boolean; - /** - * @generated from protobuf field: optional bool store_cache = 6; - */ - store_cache?: boolean; -} -// @generated message type with reflection information, may provide speed optimized methods -class SearchRequest$Type extends MessageType { - constructor() { - super("summa.proto.SearchRequest", [ - { no: 1, name: "index_alias", kind: "scalar", localName: "index_alias", T: 9 /*ScalarType.STRING*/ }, - { no: 2, name: "query", kind: "message", T: () => Query }, - { no: 3, name: "collectors", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => Collector }, - { no: 4, name: "is_fieldnorms_scoring_enabled", kind: "scalar", localName: "is_fieldnorms_scoring_enabled", opt: true, T: 8 /*ScalarType.BOOL*/ }, - { no: 5, name: "load_cache", kind: "scalar", localName: "load_cache", opt: true, T: 8 /*ScalarType.BOOL*/ }, - { no: 6, name: "store_cache", kind: "scalar", localName: "store_cache", opt: true, T: 8 /*ScalarType.BOOL*/ } - ]); - } - create(value?: PartialMessage): SearchRequest { - const message = { index_alias: "", collectors: [] }; - globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); - if (value !== undefined) - reflectionMergePartial(this, message, value); - return message; - } - internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: SearchRequest): SearchRequest { - let message = target ?? this.create(), end = reader.pos + length; - while (reader.pos < end) { - let [fieldNo, wireType] = reader.tag(); - switch (fieldNo) { - case /* string index_alias */ 1: - message.index_alias = reader.string(); - break; - case /* summa.proto.Query query */ 2: - message.query = Query.internalBinaryRead(reader, reader.uint32(), options, message.query); - break; - case /* repeated summa.proto.Collector collectors */ 3: - message.collectors.push(Collector.internalBinaryRead(reader, reader.uint32(), options)); - break; - case /* optional bool is_fieldnorms_scoring_enabled */ 4: - message.is_fieldnorms_scoring_enabled = reader.bool(); - break; - case /* optional bool load_cache */ 5: - message.load_cache = reader.bool(); - break; - case /* optional bool store_cache */ 6: - message.store_cache = reader.bool(); - break; - default: - let u = options.readUnknownField; - if (u === "throw") - throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); - let d = reader.skip(wireType); - if (u !== false) - (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); - } - } - return message; - } - internalBinaryWrite(message: SearchRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { - /* string index_alias = 1; */ - if (message.index_alias !== "") - writer.tag(1, WireType.LengthDelimited).string(message.index_alias); - /* summa.proto.Query query = 2; */ - if (message.query) - Query.internalBinaryWrite(message.query, writer.tag(2, WireType.LengthDelimited).fork(), options).join(); - /* repeated summa.proto.Collector collectors = 3; */ - for (let i = 0; i < message.collectors.length; i++) - Collector.internalBinaryWrite(message.collectors[i], writer.tag(3, WireType.LengthDelimited).fork(), options).join(); - /* optional bool is_fieldnorms_scoring_enabled = 4; */ - if (message.is_fieldnorms_scoring_enabled !== undefined) - writer.tag(4, WireType.Varint).bool(message.is_fieldnorms_scoring_enabled); - /* optional bool load_cache = 5; */ - if (message.load_cache !== undefined) - writer.tag(5, WireType.Varint).bool(message.load_cache); - /* optional bool store_cache = 6; */ - if (message.store_cache !== undefined) - writer.tag(6, WireType.Varint).bool(message.store_cache); - let u = options.writeUnknownFields; - if (u !== false) - (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); - return writer; - } -} -/** - * @generated MessageType for protobuf message summa.proto.SearchRequest - */ -export const SearchRequest = new SearchRequest$Type(); /** * @generated ServiceType for protobuf service summa.proto.SearchApi */ diff --git a/summa-wasm/src/index-registry.ts b/summa-wasm/src/index-registry.ts index de4de4e0..48460a35 100644 --- a/summa-wasm/src/index-registry.ts +++ b/summa-wasm/src/index-registry.ts @@ -1,6 +1,6 @@ import init, {setup_logging, WrappedIndexRegistry} from "../pkg"; import {IndexAttributes, IndexEngineConfig} from "./grpc-web/index_service"; -import {SearchRequest} from "./grpc-web/search_service"; +import {SearchRequest} from "./grpc-web/query"; export interface IIndexRegistry { add(index_name: string, index_engine_config: IndexEngineConfig): Promise; @@ -44,7 +44,11 @@ export class IndexRegistry implements IIndexRegistry { ) { let actual_options = Object.assign({}, default_options, options); console.log('Memory config:', actual_options.memory_config); - await init(init_url, new WebAssembly.Memory(actual_options.memory_config!)); + try { + await init(init_url, new WebAssembly.Memory(actual_options.memory_config!)); + } catch (e) { + await init(init_url + "?force", new WebAssembly.Memory(actual_options.memory_config!)); + } await setup_logging(actual_options.logging_level!); this.registry = new WrappedIndexRegistry(); } diff --git a/summa-wasm/src/remote-index-registry.ts b/summa-wasm/src/remote-index-registry.ts index 1626d456..2113b3f8 100644 --- a/summa-wasm/src/remote-index-registry.ts +++ b/summa-wasm/src/remote-index-registry.ts @@ -1,7 +1,7 @@ import * as Comlink from "comlink"; import { IndexRegistry, IIndexRegistry, IndexRegistryOptions } from "./index-registry"; import { IndexAttributes, IndexEngineConfig } from "./grpc-web/index_service"; -import { SearchRequest } from "./grpc-web/search_service"; +import {SearchRequest} from "./grpc-web/query"; export class RemoteIndexRegistry implements IIndexRegistry { init_guard: Promise; diff --git a/summa-wasm/src/service-worker.ts b/summa-wasm/src/service-worker.ts index cd2d6b0a..92e76903 100644 --- a/summa-wasm/src/service-worker.ts +++ b/summa-wasm/src/service-worker.ts @@ -66,7 +66,8 @@ async function handle_request(event: FetchEvent) { let filename = request.url; let url = request.url; let is_development = (new URL(request.url)).host == "localhost:5173" - let is_api_request = request.url.endsWith('/summa.proto.SearchApi/search'); + let is_api_request = request.url.endsWith('/summa.proto.PublicApi/search'); + let is_force = request.url.endsWith("?force"); if (is_api_request) { return fetch(event.request) @@ -113,7 +114,7 @@ async function handle_request(event: FetchEvent) { const cache = await caches.open("cache_v2"); let response = undefined; - if (caching_enabled) { + if (caching_enabled && !is_force) { response = await cache.match(url); } if (response === undefined) {