From 200ebfebdfc841750d42af68996fc5254c6da7eb Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 22 Jul 2021 16:21:42 -0700 Subject: [PATCH] fix: add 'CONTAIN' support in query builder This fix to address the issue #114 --- .../metadata/dao/utils/SearchUtils.java | 13 ++++ .../metadata/dao/search/ESSearchDAOTest.java | 2 +- .../metadata/dao/utils/SearchUtilsTest.java | 60 +++++++++++++++++++ .../metadata/dao/utils/SearchUtils.java | 14 ++++- .../metadata/dao/search/ESSearchDAOTest.java | 2 +- .../metadata/dao/utils/SearchUtilsTest.java | 59 ++++++++++++++++++ 6 files changed, 147 insertions(+), 3 deletions(-) diff --git a/dao-impl/elasticsearch-dao-7/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java b/dao-impl/elasticsearch-dao-7/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java index 769517660..f796af17f 100644 --- a/dao-impl/elasticsearch-dao-7/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java +++ b/dao-impl/elasticsearch-dao-7/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java @@ -60,6 +60,10 @@ public static Map getRequestMap(@Nullable Filter requestParams) *

If the condition between a field and value is not the same as EQUAL, a Range query is constructed. This * condition does not support multiple values for the same field. * + *

When CONTAIN, START_WITH and END_WITH conditions are used, the underlying logic is using wildcard query which is + * not performant according to ES. For details, please refer to: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-wildcard-query.html#wildcard-query-field-params + * * @param criterion {@link Criterion} single criterion which contains field, value and a comparison operator */ @Nonnull @@ -78,6 +82,15 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull Criterion crite return QueryBuilders.rangeQuery(criterion.getField()).lt(criterion.getValue().trim()); } else if (condition == Condition.LESS_THAN_OR_EQUAL_TO) { return QueryBuilders.rangeQuery(criterion.getField()).lte(criterion.getValue().trim()); + } else if (condition == Condition.CONTAIN) { + return QueryBuilders.wildcardQuery(criterion.getField(), + "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*"); + } else if (condition == Condition.START_WITH) { + return QueryBuilders.wildcardQuery(criterion.getField(), + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*"); + } else if (condition == Condition.END_WITH) { + return QueryBuilders.wildcardQuery(criterion.getField(), + "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim())); } throw new UnsupportedOperationException("Unsupported condition: " + condition); diff --git a/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java b/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java index c5d08d10d..b93de67e5 100644 --- a/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java +++ b/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java @@ -248,7 +248,7 @@ public void testFilteredQueryUnsupportedCondition() { int from = 0; int size = 10; final Filter filter2 = new Filter().setCriteria(new CriterionArray(Arrays.asList( - new Criterion().setField("field_contain").setValue("value_contain").setCondition(Condition.CONTAIN)))); + new Criterion().setField("field_contain").setValue("value_contain").setCondition(Condition.IN)))); SortCriterion sortCriterion = new SortCriterion().setOrder(SortOrder.ASCENDING).setField("urn"); assertThrows(UnsupportedOperationException.class, () -> _searchDAO.getFilteredSearchQuery(filter2, sortCriterion, from, size)); diff --git a/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java b/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java index 79f1788dc..6c8ac1841 100644 --- a/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java +++ b/dao-impl/elasticsearch-dao-7/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java @@ -7,6 +7,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; import org.testng.annotations.Test; import static com.linkedin.metadata.dao.utils.SearchUtils.*; @@ -37,4 +39,62 @@ public void testGetRequestMap() { )); assertThrows(UnsupportedOperationException.class, () -> getRequestMap(filter3)); } + + + @Test + public void testGetQueryBuilderFromContainCriterion() { + + // Given: a 'contain' criterion + Criterion containCriterion = new Criterion(); + containCriterion.setValue("match * text"); + containCriterion.setCondition(Condition.CONTAIN); + containCriterion.setField("text"); + + // Expect 'contain' criterion creates a MatchQueryBuilder + QueryBuilder queryBuilder = SearchUtils.getQueryBuilderFromCriterion(containCriterion); + assertNotNull(queryBuilder); + assertTrue(queryBuilder instanceof WildcardQueryBuilder); + + // Expect 'field name' and search terms + assertEquals(((WildcardQueryBuilder) queryBuilder).fieldName(), "text"); + assertEquals(((WildcardQueryBuilder) queryBuilder).value(), "*match \\* text*"); + } + + @Test + public void testGetQueryBuilderFromStartWithCriterion() { + + // Given: a 'start_with' criterion + Criterion containCriterion = new Criterion(); + containCriterion.setValue("match * text"); + containCriterion.setCondition(Condition.START_WITH); + containCriterion.setField("text"); + + // Expect 'start_with' criterion creates a WildcardQueryBuilder + QueryBuilder queryBuilder = SearchUtils.getQueryBuilderFromCriterion(containCriterion); + assertNotNull(queryBuilder); + assertTrue(queryBuilder instanceof WildcardQueryBuilder); + + // Expect 'field name' and search terms + assertEquals(((WildcardQueryBuilder) queryBuilder).fieldName(), "text"); + assertEquals(((WildcardQueryBuilder) queryBuilder).value(), "match \\* text*"); + } + + @Test + public void testGetQueryBuilderFromEndWithCriterion() { + + // Given: a 'end_with' criterion + Criterion containCriterion = new Criterion(); + containCriterion.setValue("match * text"); + containCriterion.setCondition(Condition.END_WITH); + containCriterion.setField("text"); + + // Expect 'end_with' criterion creates a MatchQueryBuilder + QueryBuilder queryBuilder = SearchUtils.getQueryBuilderFromCriterion(containCriterion); + assertNotNull(queryBuilder); + assertTrue(queryBuilder instanceof WildcardQueryBuilder); + + // Expect 'field name' and search terms + assertEquals(((WildcardQueryBuilder) queryBuilder).fieldName(), "text"); + assertEquals(((WildcardQueryBuilder) queryBuilder).value(), "*match \\* text"); + } } diff --git a/dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java b/dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java index 3c8ab563e..ae7fef9f2 100644 --- a/dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java +++ b/dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java @@ -68,6 +68,10 @@ static boolean isUrn(@Nonnull String value) { *

If the condition between a field and value is not the same as EQUAL, a Range query is constructed. This * condition does not support multiple values for the same field. * + *

When CONTAIN, START_WITH and END_WITH conditions are used, the underlying logic is using wildcard query which is + * not performant according to ES. For details, please refer to: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-wildcard-query.html#wildcard-query-field-params + * * @param criterion {@link Criterion} single criterion which contains field, value and a comparison operator */ @Nonnull @@ -88,8 +92,16 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull Criterion crite return QueryBuilders.rangeQuery(criterion.getField()).lt(criterion.getValue().trim()); } else if (condition == Condition.LESS_THAN_OR_EQUAL_TO) { return QueryBuilders.rangeQuery(criterion.getField()).lte(criterion.getValue().trim()); + } else if (condition == Condition.CONTAIN) { + return QueryBuilders.wildcardQuery(criterion.getField(), + "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*"); + } else if (condition == Condition.START_WITH) { + return QueryBuilders.wildcardQuery(criterion.getField(), + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*"); + } else if (condition == Condition.END_WITH) { + return QueryBuilders.wildcardQuery(criterion.getField(), + "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim())); } - throw new UnsupportedOperationException("Unsupported condition: " + condition); } diff --git a/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java b/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java index 6e18b0b76..62aae61d8 100644 --- a/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java +++ b/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/search/ESSearchDAOTest.java @@ -246,7 +246,7 @@ public void testFilteredQueryUnsupportedCondition() { int from = 0; int size = 10; final Filter filter2 = new Filter().setCriteria(new CriterionArray(Arrays.asList( - new Criterion().setField("field_contain").setValue("value_contain").setCondition(Condition.CONTAIN)))); + new Criterion().setField("field_contain").setValue("value_contain").setCondition(Condition.IN)))); SortCriterion sortCriterion = new SortCriterion().setOrder(SortOrder.ASCENDING).setField("urn"); assertThrows(UnsupportedOperationException.class, () -> _searchDAO.getFilteredSearchQuery(filter2, sortCriterion, from, size)); diff --git a/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java b/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java index 79f1788dc..c346012d6 100644 --- a/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java +++ b/dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/utils/SearchUtilsTest.java @@ -7,6 +7,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; import org.testng.annotations.Test; import static com.linkedin.metadata.dao.utils.SearchUtils.*; @@ -37,4 +39,61 @@ public void testGetRequestMap() { )); assertThrows(UnsupportedOperationException.class, () -> getRequestMap(filter3)); } + + @Test + public void testGetQueryBuilderFromContainCriterion() { + + // Given: a 'contain' criterion + Criterion containCriterion = new Criterion(); + containCriterion.setValue("match * text"); + containCriterion.setCondition(Condition.CONTAIN); + containCriterion.setField("text"); + + // Expect 'contain' criterion creates a MatchQueryBuilder + QueryBuilder queryBuilder = SearchUtils.getQueryBuilderFromCriterion(containCriterion); + assertNotNull(queryBuilder); + assertTrue(queryBuilder instanceof WildcardQueryBuilder); + + // Expect 'field name' and search terms + assertEquals(((WildcardQueryBuilder) queryBuilder).fieldName(), "text"); + assertEquals(((WildcardQueryBuilder) queryBuilder).value(), "*match \\* text*"); + } + + @Test + public void testGetQueryBuilderFromStartWithCriterion() { + + // Given: a 'start_with' criterion + Criterion containCriterion = new Criterion(); + containCriterion.setValue("match * text"); + containCriterion.setCondition(Condition.START_WITH); + containCriterion.setField("text"); + + // Expect 'start_with' criterion creates a WildcardQueryBuilder + QueryBuilder queryBuilder = SearchUtils.getQueryBuilderFromCriterion(containCriterion); + assertNotNull(queryBuilder); + assertTrue(queryBuilder instanceof WildcardQueryBuilder); + + // Expect 'field name' and search terms + assertEquals(((WildcardQueryBuilder) queryBuilder).fieldName(), "text"); + assertEquals(((WildcardQueryBuilder) queryBuilder).value(), "match \\* text*"); + } + + @Test + public void testGetQueryBuilderFromEndWithCriterion() { + + // Given: a 'end_with' criterion + Criterion containCriterion = new Criterion(); + containCriterion.setValue("match * text"); + containCriterion.setCondition(Condition.END_WITH); + containCriterion.setField("text"); + + // Expect 'end_with' criterion creates a MatchQueryBuilder + QueryBuilder queryBuilder = SearchUtils.getQueryBuilderFromCriterion(containCriterion); + assertNotNull(queryBuilder); + assertTrue(queryBuilder instanceof WildcardQueryBuilder); + + // Expect 'field name' and search terms + assertEquals(((WildcardQueryBuilder) queryBuilder).fieldName(), "text"); + assertEquals(((WildcardQueryBuilder) queryBuilder).value(), "*match \\* text"); + } }