diff --git a/package-lock.json b/package-lock.json index 58174c1e..48f1c0b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13810,7 +13810,7 @@ }, "packages/cache": { "name": "@multiversx/sdk-nestjs-cache", - "version": "3.7.8", + "version": "4.0.0", "license": "GPL-3.0-or-later", "dependencies": { "lru-cache": "^8.0.4", @@ -13849,7 +13849,7 @@ }, "packages/common": { "name": "@multiversx/sdk-nestjs-common", - "version": "3.7.8", + "version": "4.0.0", "license": "GPL-3.0-or-later", "dependencies": { "@multiversx/sdk-core": "^13.4.1", @@ -13881,7 +13881,7 @@ }, "packages/elastic": { "name": "@multiversx/sdk-nestjs-elastic", - "version": "3.7.8", + "version": "4.0.0", "license": "GPL-3.0-or-later", "devDependencies": { "@typescript-eslint/eslint-plugin": "^5.12.0", @@ -13896,7 +13896,7 @@ }, "packages/http": { "name": "@multiversx/sdk-nestjs-http", - "version": "3.7.8", + "version": "4.0.0", "license": "GPL-3.0-or-later", "dependencies": { "@multiversx/sdk-native-auth-client": "^1.0.9", @@ -13920,7 +13920,7 @@ }, "packages/monitoring": { "name": "@multiversx/sdk-nestjs-monitoring", - "version": "3.7.8", + "version": "4.0.0", "license": "GPL-3.0-or-later", "dependencies": { "prom-client": "^14.0.1", @@ -13939,7 +13939,7 @@ }, "packages/rabbitmq": { "name": "@multiversx/sdk-nestjs-rabbitmq", - "version": "3.7.8", + "version": "4.0.0", "license": "GPL-3.0-or-later", "dependencies": { "@golevelup/nestjs-rabbitmq": "4.0.0", @@ -13960,7 +13960,7 @@ }, "packages/redis": { "name": "@multiversx/sdk-nestjs-redis", - "version": "3.7.8", + "version": "4.0.0", "license": "GPL-3.0-or-later", "dependencies": { "ioredis": "^5.2.3" diff --git a/packages/cache/src/decorators/index.ts b/packages/cache/src/decorators/index.ts index 31105051..239576d1 100644 --- a/packages/cache/src/decorators/index.ts +++ b/packages/cache/src/decorators/index.ts @@ -1,2 +1 @@ export * from './no.cache'; -export * from './scrollable'; diff --git a/packages/cache/src/decorators/scrollable.ts b/packages/cache/src/decorators/scrollable.ts deleted file mode 100644 index 573caaeb..00000000 --- a/packages/cache/src/decorators/scrollable.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { DecoratorUtils } from "@multiversx/sdk-nestjs-common"; - -export class ScrollableOptions { - collection: string = ''; -} - -export const Scrollable = DecoratorUtils.registerMethodDecorator(ScrollableOptions); diff --git a/packages/cache/src/interceptors/scroll.interceptor.ts b/packages/cache/src/interceptors/scroll.interceptor.ts index a31355f2..418b66a6 100644 --- a/packages/cache/src/interceptors/scroll.interceptor.ts +++ b/packages/cache/src/interceptors/scroll.interceptor.ts @@ -1,8 +1,7 @@ -import { Constants, ContextTracker, DecoratorUtils } from "@multiversx/sdk-nestjs-common"; +import { Constants, ContextTracker, DecoratorUtils, ScrollableOptions, ScrollableCreateOptions, ScrollableAfterOptions } from "@multiversx/sdk-nestjs-common"; import { BadRequestException, CallHandler, ExecutionContext, Injectable, NestInterceptor } from "@nestjs/common"; import { Observable, catchError, tap, throwError } from "rxjs"; import { randomUUID } from "crypto"; -import { ScrollableOptions } from "../decorators"; import { CacheService } from "../cache"; @Injectable() @@ -45,19 +44,19 @@ export class ScrollInterceptor implements NestInterceptor { // create uuid and store ContextTracker.assign({ - scrollSettings: { - scrollCollection: scrollCollection, - scrollCreate: true, - }, + scrollSettings: new ScrollableCreateOptions({ + collection: scrollCollection, + create: true, + }), }); } else if (guidRegex.test(scrollCreate)) { scrollId = scrollCreate; ContextTracker.assign({ - scrollSettings: { - scrollCollection: scrollCollection, - scrollCreate: true, - }, + scrollSettings: new ScrollableCreateOptions({ + collection: scrollCollection, + create: true, + }), }); } else { throw new Error('Invalid scrollCreate value'); @@ -78,13 +77,13 @@ export class ScrollInterceptor implements NestInterceptor { } if (scrollInfo) { - ContextTracker.assign({ - scrollSettings: { - scrollCollection: scrollCollection, - scrollAfter: scrollInfo.lastSort, + ContextTracker.assign( + new ScrollableAfterOptions({ + collection: scrollCollection, + after: scrollInfo.lastSort, ids: scrollInfo.lastIds, - }, - }); + }) + ); } } else { throw new Error('Invalid scrollAfter value'); @@ -105,12 +104,12 @@ export class ScrollInterceptor implements NestInterceptor { } if (scrollInfo) { - ContextTracker.assign({ - scrollSettings: { - scrollCollection: scrollCollection, - scrollAt: scrollInfo.firstSort, - }, - }); + ContextTracker.assign( + new ScrollableAfterOptions({ + collection: scrollCollection, + after: scrollInfo.firstSort, + }) + ); } } else { throw new Error('Invalid scrollAt value'); diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index fc7bead1..3b5fe619 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -15,6 +15,7 @@ export * from './utils/url.utils'; export * from './utils/origin.logger'; export * from './utils/record.utils'; export * from './utils/round.utils'; +export * from './utils/scrollable.utils'; export * from './utils/string.utils'; export * from './utils/swagger.utils'; export * from './utils/token.utils'; diff --git a/packages/common/src/utils/scrollable.utils.ts b/packages/common/src/utils/scrollable.utils.ts new file mode 100644 index 00000000..593303dc --- /dev/null +++ b/packages/common/src/utils/scrollable.utils.ts @@ -0,0 +1,26 @@ +export class ScrollableOptions { + collection: string = ''; + + constructor(init?: Partial) { + Object.assign(this, init); + } +} + +export class ScrollableCreateOptions extends ScrollableOptions { + create: boolean = false; + + constructor(init?: Partial) { + super(init); + Object.assign(this, init); + } +} + +export class ScrollableAfterOptions extends ScrollableOptions { + after: any; + ids: any[] = []; + + constructor(init?: Partial) { + super(init); + Object.assign(this, init); + } +} diff --git a/packages/elastic/src/elastic.service.ts b/packages/elastic/src/elastic.service.ts index fd2a4837..a019b080 100644 --- a/packages/elastic/src/elastic.service.ts +++ b/packages/elastic/src/elastic.service.ts @@ -3,7 +3,7 @@ import { ApiService } from "@multiversx/sdk-nestjs-http"; import { MetricsService, ElasticMetricType, PerformanceProfiler } from "@multiversx/sdk-nestjs-monitoring"; import { ElasticQuery } from "./entities/elastic.query"; import { ElasticModuleOptions } from "./entities/elastic.module.options"; -import { ContextTracker } from "@multiversx/sdk-nestjs-common"; +import { ContextTracker, ScrollableAfterOptions } from "@multiversx/sdk-nestjs-common"; @Injectable() export class ElasticService { @@ -62,15 +62,12 @@ export class ElasticService { private async getListResult(url: string, collection: string, elasticQuery: ElasticQuery) { const scrollSettings = ContextTracker.get()?.scrollSettings; - const elasticQueryJson: any = elasticQuery.toJson(); - if (scrollSettings && scrollSettings.scrollCollection === collection) { + if (scrollSettings && scrollSettings.collection === collection) { let documents: any[] = []; - if (scrollSettings.scrollAfter) { - documents = await this.getScrollAfterResult(url, elasticQuery, scrollSettings.scrollAfter, scrollSettings.ids, elasticQuery.pagination?.size ?? 25); - } else if (scrollSettings.scrollAt) { - documents = await this.getScrollAtResult(url, elasticQuery, scrollSettings.scrollAt); - } else if (scrollSettings.scrollCreate) { + if (scrollSettings.after) { + documents = await this.getScrollAfterResult(url, elasticQuery, scrollSettings); + } else if (scrollSettings.create) { documents = await this.getScrollCreateResult(url, elasticQuery); } else { throw new Error('Invalid scroll settings'); @@ -81,6 +78,7 @@ export class ElasticService { return documents; } + const elasticQueryJson: any = elasticQuery.toJson(); const result = await this.post(url, elasticQueryJson); return result.data.hits.hits; } @@ -111,51 +109,34 @@ export class ElasticService { }); } - private async getScrollAtResult(url: string, elasticQuery: ElasticQuery, scrollAt: any) { - const elasticQueryJson: any = elasticQuery.toJson(); - elasticQueryJson.search_after = scrollAt; - - const result = await this.post(url, elasticQueryJson); - return result.data.hits.hits; - } - - private async getScrollAfterResult(url: string, elasticQuery: ElasticQuery, scrollAfter: any, ids: string[], size: number) { - const MAX_SIZE = 10000; - + private async getScrollAfterResult(url: string, elasticQuery: ElasticQuery, scrollSettings: ScrollableAfterOptions) { const elasticQueryJson: any = elasticQuery.toJson(); - elasticQueryJson.search_after = scrollAfter; - elasticQueryJson.size += ids.length; - - let remainingSize = 0; - if (elasticQueryJson.size > MAX_SIZE) { - remainingSize = elasticQueryJson.size - MAX_SIZE; - elasticQueryJson.size = MAX_SIZE; - } + elasticQueryJson.search_after = scrollSettings.after; + this.excludeIds(elasticQueryJson, scrollSettings.ids); const queryResult = await this.post(url, elasticQueryJson); - const allDocuments = this.excludeIds(queryResult.data.hits.hits, ids, elasticQuery.pagination?.size); - - let result = allDocuments; - - if (remainingSize && allDocuments.length > 0) { - ids = this.getLastIds(allDocuments); - const lastDocumentSort = allDocuments[allDocuments.length - 1].sort; - - elasticQueryJson.size = remainingSize + ids.length; - elasticQueryJson.search_after = lastDocumentSort; + return queryResult.data.hits.hits; + } - const remainingResult = await this.post(url, elasticQueryJson); - const remainingDocuments = this.excludeIds(remainingResult.data.hits.hits, ids, remainingSize); + private excludeIds(elasticQueryJson: any, ids: any[]) { + if (!elasticQueryJson.query) { + elasticQueryJson.query = {}; + } - result = allDocuments.concat(remainingDocuments); + if (!elasticQueryJson.query.bool) { + elasticQueryJson.query.bool = {}; } - return result.slice(0, size); - } + if (!elasticQueryJson.query.bool.must_not) { + elasticQueryJson.query.bool.must_not = []; + } - private excludeIds(documents: any[], ids: string[], maxSize: number | undefined) { - return documents.filter((document: any) => !ids.includes(document._id)).slice(0, maxSize ?? 25); + elasticQueryJson.query.bool.must_not.push({ + terms: { + _id: ids, + }, + }); } async getList(collection: string, key: string, elasticQuery: ElasticQuery, overrideUrl?: string): Promise {