diff --git a/modules/library/common-search/src/main/resources/application.yml b/modules/library/common-search/src/main/resources/application.yml index c17007ae..84e1cb52 100644 --- a/modules/library/common-search/src/main/resources/application.yml +++ b/modules/library/common-search/src/main/resources/application.yml @@ -3,7 +3,7 @@ gn: # Full text on all fields # 'queryBase': '${any}', # Full text but more boost on title match - queryFilter: '+isTemplate:n AND -indexingError:true' + queryFilter: '+isTemplate:n' queryBase: '(any.\*:(${any}) OR resourceTitleObject.\*:(${any})^2)' trackTotalHits: true scoreConfig: > diff --git a/modules/services/indexing/pom.xml b/modules/services/indexing/pom.xml index c4cc06f2..7d0d870c 100644 --- a/modules/services/indexing/pom.xml +++ b/modules/services/indexing/pom.xml @@ -97,12 +97,8 @@ camel-micrometer-starter - org.elasticsearch.client - elasticsearch-rest-high-level-client - - - org.elasticsearch - elasticsearch + co.elastic.clients + elasticsearch-java diff --git a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java index 7bf2d3f7..5302066e 100644 --- a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java +++ b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java @@ -45,7 +45,7 @@ public ResponseEntity indexAll( @PostMapping("/event/{bucket}/{uuid}") public void sendEvent( @PathVariable("bucket")String bucket, - @PathVariable("uuid") String uuid) throws Exception { + @PathVariable("uuid") String uuid) { eventStreamService.produceEvent(bucket, uuid); } } diff --git a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java index ed0f4b29..5d3f501f 100644 --- a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java +++ b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java @@ -6,9 +6,14 @@ package org.fao.geonet.indexing.service; -import static org.elasticsearch.rest.RestStatus.CREATED; -import static org.elasticsearch.rest.RestStatus.OK; - +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.AcknowledgedResponse; +import co.elastic.clients.elasticsearch._types.Refresh; +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.IndexRequest; +import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; @@ -28,15 +33,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.camel.Exchange; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.xcontent.XContentType; import org.fao.geonet.common.xml.XsltUtil; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.Metadata; @@ -63,16 +60,19 @@ public class IndexingService { MetadataRepository metadataRepository; @Autowired - RestHighLevelClient client; + ElasticsearchClient client; /** * Delete index. */ public void deleteIndex(Exchange e) { try { + DeleteIndexRequest deleteIndexRequest = DeleteIndexRequest.of( + b -> b.index(index) + ); AcknowledgedResponse deleteIndexResponse = client.indices() - .delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT); - if (deleteIndexResponse.isAcknowledged()) { + .delete(deleteIndexRequest); + if (deleteIndexResponse.acknowledged()) { log.info(String.format( "Index %s removed.", index)); @@ -202,20 +202,26 @@ protected static String collectDbProperties(AbstractMetadata r) { private BulkRequest buildBulkRequest(IndexRecords indexRecords) { - BulkRequest bulkRequest = new BulkRequest(index); + BulkRequest.Builder requestBuilder = new BulkRequest.Builder() + .index(index) + .refresh(Refresh.True); ObjectMapper mapper = new ObjectMapper(); indexRecords.getIndexRecord().forEach(r -> { try { - IndexRequest indexRequest = new IndexRequest(); - indexRequest.id(r.getId()); - indexRequest.source(mapper.writeValueAsString(r), XContentType.JSON); - bulkRequest.add(indexRequest); + String json = mapper.writeValueAsString(r); + requestBuilder.operations(op -> op + .index(idx -> idx + .index(index) + .id(r.getId()) + .document(json) + ) + ); } catch (JsonProcessingException jsonProcessingException) { jsonProcessingException.printStackTrace(); } }); - return bulkRequest; + return requestBuilder.build(); } private void sendToIndex(IndexRecords indexRecords, @@ -223,16 +229,16 @@ private void sendToIndex(IndexRecords indexRecords, BulkRequest bulkRequest = buildBulkRequest(indexRecords); try { // TODO: Asynchronous? - BulkResponse bulkItemResponses = client.bulk(bulkRequest, RequestOptions.DEFAULT); + BulkResponse bulkItemResponses = client.bulk(bulkRequest); log.info(String.format( "Indexing operation took %d.", - bulkItemResponses.getIngestTookInMillis() + bulkItemResponses.took() )); - if (bulkItemResponses.hasFailures()) { + if (bulkItemResponses.errors()) { AtomicInteger failureCount = new AtomicInteger(); - Arrays.stream(bulkItemResponses.getItems()).forEach(item -> { - if (item.status() != OK - && item.status() != CREATED) { + bulkItemResponses.items().forEach(item -> { + if (item.status() != 200 + && item.status() != 201) { failureCount.getAndIncrement(); // TODO: Index error document } @@ -243,12 +249,6 @@ private void sendToIndex(IndexRecords indexRecords, failureCount )); } - } catch (ElasticsearchStatusException indexException) { - report.setNumberOfRecordsWithIndexingErrors(indexRecords.getIndexRecord().size()); - log.error(String.format( - "Error while saving records %d in index. Error is: %s.", - indexException.getMessage() - )); } catch (IOException ioException) { log.error(String.format( "Error while sending records to index. Error is: %s.", diff --git a/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java b/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java index 435ba206..e15dc2b2 100644 --- a/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java +++ b/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java @@ -19,6 +19,7 @@ import org.fao.geonet.ogcapi.records.MvcConfigurer; import org.fao.geonet.repository.LanguageRepository; import org.fao.geonet.repository.MetadataRepository; +import org.fao.geonet.repository.SettingRepository; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.UiSettingsRepository; import org.fao.geonet.view.ViewUtility; @@ -64,6 +65,9 @@ public class ItemApiControllerTest { @MockBean private MetadataRepository mockMetadataRepository; + @MockBean + private SettingRepository mockSettingRepository; + @Autowired private MockMvc mockMvc; diff --git a/modules/services/searching/pom.xml b/modules/services/searching/pom.xml index 7a113cc3..fe172cb9 100644 --- a/modules/services/searching/pom.xml +++ b/modules/services/searching/pom.xml @@ -93,11 +93,6 @@ lombok - - org.springdoc springdoc-openapi-common diff --git a/pom.xml b/pom.xml index 69befa2f..b6ef4c0e 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ 4.2.7-0 9.4.27.v20200227 - 7.15.1 + 8.11.3 1.5.5 3.5.0 2.1.2 @@ -213,13 +213,8 @@ - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - org.elasticsearch - elasticsearch + co.elastic.clients + elasticsearch-java ${elasticsearch.version}