feat(es): move simple filters to the filter array to save performance

see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html#query-dsl-bool-query "The clause (query) must appear in matching documents. However unlike must the score of the query will be ignored. Filter clauses are executed in filter context, meaning that scoring is ignored and clauses are considered for caching."
geonetwork · Oct 5, 2023 · efab617 · efab617
1 parent eb627f1
commit efab617
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 67 deletions.
diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts
@@ -107,14 +107,20 @@ describe('ElasticsearchService', () => {
 
       expect(query).toEqual({
         bool: {
-          filter: [],
-          should: [],
-          must: [
+          filter: [
             {
               terms: {
                 isTemplate: ['n'],
               },
             },
+            {
+              ids: {
+                values: ['record-1', 'record-2', 'record-3'],
+              },
+            },
+          ],
+          should: [],
+          must: [
             {
               query_string: {
                 default_operator: 'AND',
@@ -129,11 +135,6 @@ describe('ElasticsearchService', () => {
                 query: 'hello',
               },
             },
-            {
-              ids: {
-                values: ['record-1', 'record-2', 'record-3'],
-              },
-            },
           ],
           must_not: {
             terms: {
@@ -156,14 +157,25 @@ describe('ElasticsearchService', () => {
       )
       expect(query).toEqual({
         bool: {
-          filter: [],
-          should: [],
-          must: [
+          filter: [
             {
               terms: {
                 isTemplate: ['n'],
               },
             },
+            {
+              query_string: {
+                query: 'Org:("world")',
+              },
+            },
+            {
+              ids: {
+                values: ['record-1', 'record-2', 'record-3'],
+              },
+            },
+          ],
+          should: [],
+          must: [
             {
               query_string: {
                 default_operator: 'AND',
@@ -178,16 +190,6 @@ describe('ElasticsearchService', () => {
                 query: 'hello',
               },
             },
-            {
-              query_string: {
-                query: 'Org:("world")',
-              },
-            },
-            {
-              ids: {
-                values: ['record-1', 'record-2', 'record-3'],
-              },
-            },
           ],
           must_not: {
             terms: {
@@ -214,14 +216,25 @@ describe('ElasticsearchService', () => {
       )
       expect(query).toEqual({
         bool: {
-          filter: [],
-          should: [],
-          must: [
+          filter: [
             {
               terms: {
                 isTemplate: ['n'],
               },
             },
+            {
+              query_string: {
+                query: 'Org:("world" OR "world2") AND name:("john")',
+              },
+            },
+            {
+              ids: {
+                values: [],
+              },
+            },
+          ],
+          should: [],
+          must: [
             {
               query_string: {
                 default_operator: 'AND',
@@ -236,16 +249,6 @@ describe('ElasticsearchService', () => {
                 query: 'hello',
               },
             },
-            {
-              query_string: {
-                query: 'Org:("world" OR "world2") AND name:("john")',
-              },
-            },
-            {
-              ids: {
-                values: [],
-              },
-            },
           ],
           must_not: {
             terms: {
@@ -269,9 +272,7 @@ describe('ElasticsearchService', () => {
       )
       expect(query).toEqual({
         bool: {
-          filter: [],
-          should: [],
-          must: [
+          filter: [
             {
               terms: {
                 isTemplate: ['n'],
@@ -288,6 +289,8 @@ describe('ElasticsearchService', () => {
               },
             },
           ],
+          should: [],
+          must: [],
           must_not: {
             terms: {
               resourceType: ['service', 'map', 'map/static', 'mapDigital'],
@@ -307,14 +310,25 @@ describe('ElasticsearchService', () => {
       )
       expect(query).toEqual({
         bool: {
-          filter: [],
-          should: [],
-          must: [
+          filter: [
             {
               terms: {
                 isTemplate: ['n'],
               },
             },
+            {
+              query_string: {
+                query: 'Org:(world AND world2)',
+              },
+            },
+            {
+              ids: {
+                values: [],
+              },
+            },
+          ],
+          should: [],
+          must: [
             {
               query_string: {
                 default_operator: 'AND',
@@ -329,16 +343,6 @@ describe('ElasticsearchService', () => {
                 query: 'hello',
               },
             },
-            {
-              query_string: {
-                query: 'Org:(world AND world2)',
-              },
-            },
-            {
-              ids: {
-                values: [],
-              },
-            },
           ],
           must_not: {
             terms: {
@@ -360,7 +364,7 @@ describe('ElasticsearchService', () => {
         )
       })
       it('escapes special char', () => {
-        expect(query.bool.must[1].query_string.query).toEqual(
+        expect(query.bool.must[0].query_string.query).toEqual(
           `scot \\(\\)\\{\\?\\[ \\/ test`
         )
       })
@@ -392,13 +396,19 @@ describe('ElasticsearchService', () => {
         )
         expect(query).toEqual({
           bool: {
-            filter: [],
-            must: [
+            filter: [
               {
                 terms: {
                   isTemplate: ['n'],
                 },
               },
+              {
+                query_string: {
+                  query: 'Org:(world)',
+                },
+              },
+            ],
+            must: [
               {
                 query_string: {
                   default_operator: 'AND',
@@ -413,11 +423,6 @@ describe('ElasticsearchService', () => {
                   query: 'hello',
                 },
               },
-              {
-                query_string: {
-                  query: 'Org:(world)',
-                },
-              },
             ],
             must_not: {
               terms: {

diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts
@@ -208,10 +208,7 @@ export class ElasticsearchService {
     uuids?: string[],
     geometry?: Geometry
   ) {
-    const must = [this.queryFilterOnValues('isTemplate', 'n')] as Record<
-      string,
-      unknown
-    >[]
+    const must = [] as Record<string, unknown>[]
     const must_not = {
       ...this.queryFilterOnValues('resourceType', [
         'service',
@@ -221,6 +218,10 @@ export class ElasticsearchService {
       ]),
     }
     const should = [] as Record<string, unknown>[]
+    const filter = [this.queryFilterOnValues('isTemplate', 'n')] as Record<
+      string,
+      unknown
+    >[]
 
     if (any) {
       must.push({
@@ -236,14 +237,14 @@ export class ElasticsearchService {
     }
     const queryFilters = this.filtersToQueryString(fieldSearchFilters)
     if (queryFilters) {
-      must.push({
+      filter.push({
         query_string: {
           query: queryFilters,
         },
       })
     }
     if (uuids) {
-      must.push({
+      filter.push({
         ids: {
           values: uuids,
         },
@@ -277,7 +278,7 @@ export class ElasticsearchService {
         must,
         must_not,
         should,
-        filter: [],
+        filter,
       },
     }
   }

diff --git a/libs/api/repository/src/lib/gn4/organizations/organizations-from-metadata.service.spec.ts b/libs/api/repository/src/lib/gn4/organizations/organizations-from-metadata.service.spec.ts
@@ -270,7 +270,7 @@ describe.each(['4.2.2-00', '4.2.3-xx', '4.2.5-xx'])(
               size: 0,
               query: {
                 bool: {
-                  must: [{ terms: { isTemplate: ['n'] } }],
+                  must: [],
                   must_not: {
                     terms: {
                       resourceType: [
@@ -282,7 +282,7 @@ describe.each(['4.2.2-00', '4.2.3-xx', '4.2.5-xx'])(
                     },
                   },
                   should: [],
-                  filter: [],
+                  filter: [{ terms: { isTemplate: ['n'] } }],
                 },
               },
               _source: [],