From bac3b8e83c095b5182a56699494392844849e21c Mon Sep 17 00:00:00 2001 From: Holash Chand Date: Fri, 10 Nov 2023 16:38:01 +0530 Subject: [PATCH 1/2] fixed search queries, es equals and endsWith, native search notContains query --- .../java/dev/sunbirdrc/elastic/ElasticServiceImpl.java | 4 ++-- .../java/dev/sunbirdrc/registry/dao/SearchDaoImpl.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java b/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java index 75815048d..d866103ce 100644 --- a/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java +++ b/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java @@ -335,7 +335,7 @@ private BoolQueryBuilder buildQuery(SearchQuery searchQuery) { } switch (operator) { case eq: - query = query.must(QueryBuilders.matchPhraseQuery(field, value)); + query = query.must(QueryBuilders.matchPhraseQuery(String.format("%s.keyword", field), value)); break; case neq: query = query.mustNot(QueryBuilders.matchPhraseQuery(field, value)); @@ -369,7 +369,7 @@ private BoolQueryBuilder buildQuery(SearchQuery searchQuery) { query = query.must(QueryBuilders.matchPhrasePrefixQuery(field, value.toString())); break; case endsWith: - query = query.must(QueryBuilders.wildcardQuery(field, "*" + value)); + query = query.must(QueryBuilders.wildcardQuery(String.format("%s.keyword", field), "*" + value)); break; case notContains: query = query.mustNot(QueryBuilders.matchPhraseQuery(field, value)); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/SearchDaoImpl.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/SearchDaoImpl.java index f40e09eb6..806d06078 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/SearchDaoImpl.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/SearchDaoImpl.java @@ -100,22 +100,22 @@ private GraphTraversal getFilteredResultTraversal( break; case contains: - condition = (s1, s2) -> (s1.contains(s2)); + condition = String::contains; resultGraphTraversal = resultGraphTraversal.has(property, new P(condition, genericValue.toString())); break; case startsWith: - condition = (s1, s2) -> (s1.startsWith(s2)); + condition = String::startsWith; resultGraphTraversal = resultGraphTraversal.has(property, new P(condition, genericValue.toString())); break; case endsWith: - condition = (s1, s2) -> (s1.endsWith(s2)); + condition = String::endsWith; resultGraphTraversal = resultGraphTraversal.has(property, new P(condition, genericValue.toString())); break; case notContains: - condition = (s1, s2) -> (s1.contains(s2)); + condition = (s1, s2) -> (!s1.contains(s2)); resultGraphTraversal = resultGraphTraversal.has(property, new P(condition, genericValue.toString())); break; From d5929b124c96f6e67fb712b9d0fb983fa9ded19d Mon Sep 17 00:00:00 2001 From: Holash Chand Date: Fri, 19 Jan 2024 14:31:05 +0530 Subject: [PATCH 2/2] added unit and e2e tests for the changes on search operators --- Makefile | 2 +- .../java/e2e/registry/TeacherRequest2.json | 6 ++ .../test/java/e2e/registry/registry.feature | 20 +++++- .../registry/dao/impl/SearchDaoImplTest.java | 67 +++++++++++++++---- ..._with_asyncCreate_events_notifications.env | 1 + 5 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 java/apitest/src/test/java/e2e/registry/TeacherRequest2.json diff --git a/Makefile b/Makefile index fc4c3934f..12bfaa670 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test: build @docker-compose down @rm -rf db-data-1 || echo "no permission to delete" # test with kafka(async), events, notifications, - @docker-compose --env-file test_environments/test_with_asyncCreate_events_notifications.env up -d db clickhouse redis keycloak registry certificate-signer certificate-api kafka zookeeper notification-ms metrics + @docker-compose --env-file test_environments/test_with_asyncCreate_events_notifications.env up -d db es clickhouse redis keycloak registry certificate-signer certificate-api kafka zookeeper notification-ms metrics @echo "Starting the test" && sh build/wait_for_port.sh 8080 @echo "Starting the test" && sh build/wait_for_port.sh 8081 @docker-compose ps diff --git a/java/apitest/src/test/java/e2e/registry/TeacherRequest2.json b/java/apitest/src/test/java/e2e/registry/TeacherRequest2.json new file mode 100644 index 000000000..26ce338ea --- /dev/null +++ b/java/apitest/src/test/java/e2e/registry/TeacherRequest2.json @@ -0,0 +1,6 @@ +{ + "name": "abc def", + "gender": "male", + "contact": "1238", + "email": "u73@mail.com" +} \ No newline at end of file diff --git a/java/apitest/src/test/java/e2e/registry/registry.feature b/java/apitest/src/test/java/e2e/registry/registry.feature index 90f96af73..dd8eb228f 100644 --- a/java/apitest/src/test/java/e2e/registry/registry.feature +++ b/java/apitest/src/test/java/e2e/registry/registry.feature @@ -224,10 +224,28 @@ Feature: Registry api tests Then status 200 And response.result.Teacher.transactionId.length > 0 * sleep(7000) + # create entity for teacher + Given url registryUrl + And path 'api/v1/Teacher?mode=async' + * def teacherRequest = read('TeacherRequest2.json') + And request teacherRequest + When method post + Then status 200 + And response.result.Teacher.transactionId.length > 0 + * sleep(7000) # get teacher info Given url registryUrl And path 'api/v1/Teacher/search' - And request {"filters":{}} + And request {"filters":{ "name": { "eq": "abc" }}} + When method post + Then status 200 + * print response + And response.length == 1 + And response[0].contact == '#notpresent' + # get teacher info + Given url registryUrl + And path 'api/v1/Teacher/search' + And request {"filters":{ "name": { "endsWith": "abc" }}} When method post Then status 200 * print response diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java index afc81504f..96845ed25 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java @@ -1,7 +1,5 @@ package dev.sunbirdrc.registry.dao.impl; -import static org.junit.Assert.assertTrue; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import dev.sunbirdrc.pojos.Filter; @@ -27,6 +25,7 @@ import javax.annotation.PreDestroy; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.hamcrest.core.Every; import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; @@ -37,6 +36,8 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; +import static org.junit.Assert.*; + @RunWith(SpringRunner.class) @SpringBootTest(classes = { DefinitionsManager.class, ObjectMapper.class, DBProviderFactory.class, DBConnectionInfoMgr.class, OSResourceLoader.class }) @FixMethodOrder(MethodSorters.NAME_ASCENDING) @@ -79,21 +80,25 @@ public void initializeGraph() throws IOException { public void test_search_no_response() throws AuditFailedException, EncryptionException, RecordNotFoundException { SearchQuery searchQuery = getSearchQuery(entities, "", "", FilterOperators.eq);//new SearchQuery("", 0, 0); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 0); + assertTrue(result.get("Teacher").isEmpty()); } @Test public void testEqOperator() { SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "marko", FilterOperators.eq); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 1); + result.get("Teacher").forEach(d -> { + assertEquals("marko", d.get("teacherName").asText()); + }); } @Test public void testNeqOperator() { SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "marko", FilterOperators.neq); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 2); + result.get("Teacher").forEach(d -> { + assertNotEquals("marko", d.get("teacherName").asText()); + }); } @Test @@ -103,7 +108,10 @@ public void testRangeOperator() { range.add(3); SearchQuery searchQuery = getSearchQuery(entities, "serialNum", range, FilterOperators.between); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 2); + result.get("Teacher").forEach(d -> { + assertTrue(d.get("serialNum").asLong() >= 1); + assertTrue(d.get("serialNum").asLong() <= 3); + }); } @Test @@ -114,33 +122,60 @@ public void testOrOperator() { values.add(VALUE_NOT_PRESENT); SearchQuery searchQuery = getSearchQuery(entities, "teacherName", values, FilterOperators.or); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 2); + result.get("Teacher").forEach(d -> { + assertTrue(values.contains(d.get("teacherName").asText())); + }); } @Test public void testStartsWithOperator() { SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "ma", FilterOperators.startsWith); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 1); + result.get("Teacher").forEach(d -> { + assertTrue(d.get("teacherName").asText().startsWith("ma")); + }); } @Test public void testNotStartsWithOperator() { SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "ma", FilterOperators.notStartsWith); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 2); + result.get("Teacher").forEach(d -> { + assertFalse(d.get("teacherName").asText().startsWith("ma")); + }); } @Test public void testEndsWithOperator() { SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "as", FilterOperators.endsWith); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 2); + result.get("Teacher").forEach(d -> { + assertTrue(d.get("teacherName").asText().endsWith("as")); + }); } @Test public void testNotEndsWithOperator() { SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "as", FilterOperators.notEndsWith); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 1); + result.get("Teacher").forEach(d -> { + assertFalse(d.get("teacherName").asText().endsWith("as")); + }); + } + + @Test + public void testContainsOperator() { + SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "as", FilterOperators.contains); + JsonNode result = searchDao.search(graph, searchQuery, expandInternal); + result.get("Teacher").forEach(d -> { + assertTrue(d.get("teacherName").asText().contains("as")); + }); + } + @Test + public void testNotContainsOperator() { + SearchQuery searchQuery = getSearchQuery(entities, "teacherName", "as", FilterOperators.notContains); + JsonNode result = searchDao.search(graph, searchQuery, expandInternal); + result.get("Teacher").forEach(d -> { + assertFalse(d.get("teacherName").asText().contains("as")); + }); } @Test public void testMultiOperators() { @@ -152,14 +187,14 @@ public void testMultiOperators() { searchQuery.getFilters().add(new Filter("serialNum", FilterOperators.gt, 1)); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 0); + assertTrue(result.get("Teacher").isEmpty()); } @Test public void testResponseLimit() { SearchQuery searchQuery = new SearchQuery(entities, offset, limit); JsonNode result = searchDao.search(graph, searchQuery, expandInternal); - assertTrue(result.get("Teacher").size() == 1); + assertEquals(1, result.get("Teacher").size()); } @PreDestroy @@ -187,6 +222,12 @@ private void populateGraph() { Vertex v3 = vertexWriter.createVertex("Teacher"); v3.property("serialNum", 3); v3.property("teacherName", "jas"); + Vertex v4 = vertexWriter.createVertex("Teacher"); + v4.property("serialNum", 4); + v4.property("teacherName", "markus marko"); + Vertex v5 = vertexWriter.createVertex("Teacher"); + v5.property("serialNum", 5); + v5.property("teacherName", "sodi jas jakob"); } private SearchQuery getSearchQuery(SearchQuery searchQuery, Filter filter, String type) { diff --git a/test_environments/test_with_asyncCreate_events_notifications.env b/test_environments/test_with_asyncCreate_events_notifications.env index 18e661aec..24ace3b00 100644 --- a/test_environments/test_with_asyncCreate_events_notifications.env +++ b/test_environments/test_with_asyncCreate_events_notifications.env @@ -8,6 +8,7 @@ TRACK_NOTIFICATIONS=true EVENT_ENABLED=true ASYNC_ENABLED=true RELEASE_VERSION=latest +SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.ElasticSearchService NOTIFICATION_URL=http://notification-ms:8765/notification-service/v1/notification KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889