Skip to content

Commit

Permalink
Handling duplicate selections in MongoDB (#66)
Browse files Browse the repository at this point in the history 
* Handle duplications using
(i) a overwrite strategy for duplicate selections
(ii) an error-out strategy for duplicate aliases
  • Loading branch information
@suresh-prakash
suresh-prakash authored Dec 30, 2021
1 parent 506300c commit 0b23b9a
Show file tree
Hide file tree
Showing 4 changed files with 178 additions and 3 deletions.
2 changes: 1 addition & 1 deletion document-store/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation("org.apache.commons:commons-collections4:4.4")
implementation("org.postgresql:postgresql:42.2.13")
implementation("org.mongodb:mongodb-driver-sync:4.1.2")
implementation("com.fasterxml.jackson.core:jackson-databind:2.11.0")
implementation("com.fasterxml.jackson.core:jackson-databind:2.12.6")
implementation("org.slf4j:slf4j-api:1.7.32")
implementation("com.google.guava:guava-annotations:r03")
implementation("org.apache.commons:commons-lang3:3.10")
Expand Down
118 changes: 118 additions & 0 deletions ...nTest/java/org/hypertrace/core/documentstore/mongo/MongoQueryExecutorIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,32 @@ public void testFindSimple() throws IOException {
assertDocsEqual(resultDocs, "mongo/simple_filter_response.json");
}

@Test
public void testFindWithDuplicateSelections() throws IOException {
List<SelectionSpec> selectionSpecs =
List.of(
SelectionSpec.of(IdentifierExpression.of("item")),
SelectionSpec.of(IdentifierExpression.of("item")),
SelectionSpec.of(IdentifierExpression.of("price")),
SelectionSpec.of(IdentifierExpression.of("quantity")),
SelectionSpec.of(IdentifierExpression.of("quantity")),
SelectionSpec.of(IdentifierExpression.of("date")));
Selection selection = Selection.builder().selectionSpecs(selectionSpecs).build();
Filter filter =
Filter.builder()
.expression(
RelationalExpression.of(
IdentifierExpression.of("item"),
NOT_IN,
ConstantExpression.ofStrings(List.of("Soap", "Bottle"))))
.build();

Query query = Query.builder().setSelection(selection).setFilter(filter).build();

Iterator<Document> resultDocs = collection.find(query);
assertDocsEqual(resultDocs, "mongo/simple_filter_response.json");
}

@Test
public void testFindWithSortingAndPagination() throws IOException {
List<SelectionSpec> selectionSpecs =
Expand Down Expand Up @@ -166,6 +192,47 @@ public void testFindWithSortingAndPagination() throws IOException {
assertDocsEqual(resultDocs, "mongo/filter_with_sorting_and_pagination_response.json");
}

@Test
public void testFindWithDuplicateSortingAndPagination() throws IOException {
List<SelectionSpec> selectionSpecs =
List.of(
SelectionSpec.of(IdentifierExpression.of("item")),
SelectionSpec.of(IdentifierExpression.of("price")),
SelectionSpec.of(IdentifierExpression.of("quantity")),
SelectionSpec.of(IdentifierExpression.of("date")));
Selection selection = Selection.builder().selectionSpecs(selectionSpecs).build();

Filter filter =
Filter.builder()
.expression(
RelationalExpression.of(
IdentifierExpression.of("item"),
IN,
ConstantExpression.ofStrings(List.of("Mirror", "Comb", "Shampoo", "Bottle"))))
.build();

Sort sort =
Sort.builder()
.sortingSpec(SortingSpec.of(IdentifierExpression.of("quantity"), DESC))
.sortingSpec(SortingSpec.of(IdentifierExpression.of("item"), ASC))
.sortingSpec(SortingSpec.of(IdentifierExpression.of("quantity"), DESC))
.sortingSpec(SortingSpec.of(IdentifierExpression.of("item"), ASC))
.build();

Pagination pagination = Pagination.builder().offset(1).limit(3).build();

Query query =
Query.builder()
.setSelection(selection)
.setFilter(filter)
.setSort(sort)
.setPagination(pagination)
.build();

Iterator<Document> resultDocs = collection.find(query);
assertDocsEqual(resultDocs, "mongo/filter_with_sorting_and_pagination_response.json");
}

@Test
public void testFindWithNestedFields() throws IOException {
List<SelectionSpec> selectionSpecs =
Expand Down Expand Up @@ -221,6 +288,18 @@ public void testAggregateSimple() throws IOException {
assertDocsEqual(resultDocs, "mongo/count_response.json");
}

@Test
public void testAggregateWithDuplicateSelections() throws IOException {
Query query =
Query.builder()
.addSelection(AggregateExpression.of(COUNT, IdentifierExpression.of("item")), "count")
.addSelection(AggregateExpression.of(COUNT, IdentifierExpression.of("item")), "count")
.build();

Iterator<Document> resultDocs = collection.aggregate(query);
assertDocsEqual(resultDocs, "mongo/count_response.json");
}

@Test
public void testAggregateWithFiltersAndOrdering() throws IOException {
Query query =
Expand Down Expand Up @@ -257,6 +336,45 @@ public void testAggregateWithFiltersAndOrdering() throws IOException {
assertDocsEqual(resultDocs, "mongo/sum_response.json");
}

@Test
public void testAggregateWithFiltersAndDuplicateOrderingAndDuplicateAggregations()
throws IOException {
Query query =
Query.builder()
.addSelection(
AggregateExpression.of(
SUM,
FunctionExpression.builder()
.operand(IdentifierExpression.of("price"))
.operator(MULTIPLY)
.operand(IdentifierExpression.of("quantity"))
.build()),
"total")
.addSelection(IdentifierExpression.of("item"))
.addAggregation(IdentifierExpression.of("item"))
.addAggregation(IdentifierExpression.of("item"))
.addSort(IdentifierExpression.of("total"), DESC)
.addSort(IdentifierExpression.of("total"), DESC)
.setAggregationFilter(
LogicalExpression.builder()
.operand(
RelationalExpression.of(
IdentifierExpression.of("total"), GTE, ConstantExpression.of(11)))
.operator(AND)
.operand(
RelationalExpression.of(
IdentifierExpression.of("total"), LTE, ConstantExpression.of(99)))
.build())
.setFilter(
RelationalExpression.of(
IdentifierExpression.of("quantity"), NEQ, ConstantExpression.of(10)))
.setPagination(Pagination.builder().limit(10).offset(0).build())
.build();

Iterator<Document> resultDocs = collection.aggregate(query);
assertDocsEqual(resultDocs, "mongo/sum_response.json");
}

@Test
public void testAggregateWithNestedFields() throws IOException {
Query query =
Expand Down
16 changes: 15 additions & 1 deletion ...n/java/org/hypertrace/core/documentstore/mongo/parser/MongoSelectingExpressionParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,11 @@ public static BasicDBObject getSelections(final Query query) {
selectionSpecs.stream()
.map(spec -> MongoSelectingExpressionParser.parse(parser, spec))
.flatMap(map -> map.entrySet().stream())
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
.collect(
toMap(
Map.Entry::getKey,
Map.Entry::getValue,
MongoSelectingExpressionParser::mergeValues));

return new BasicDBObject(projectionMap);
}
Expand All @@ -79,4 +83,14 @@ private static Map<String, Object> parse(

return spec.getExpression().visit(parser);
}

private static <T> T mergeValues(final T first, final T second) {
if (first.equals(second)) {
return second;
}

throw new IllegalArgumentException(
String.format(
"Query contains duplicate aliases with different selections: (%s, %s)", first, second));
}
}
45 changes: 44 additions & 1 deletion ...t-store/src/test/java/org/hypertrace/core/documentstore/mongo/MongoQueryExecutorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NOT_IN;
import static org.hypertrace.core.documentstore.expression.operators.SortingOrder.ASC;
import static org.hypertrace.core.documentstore.expression.operators.SortingOrder.DESC;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyList;
Expand All @@ -37,8 +38,10 @@
import org.hypertrace.core.documentstore.expression.impl.LogicalExpression;
import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
import org.hypertrace.core.documentstore.expression.operators.SortingOrder;
import org.hypertrace.core.documentstore.query.Filter;
import org.hypertrace.core.documentstore.query.Pagination;
import org.hypertrace.core.documentstore.query.Query;
import org.hypertrace.core.documentstore.query.Selection;
import org.hypertrace.core.documentstore.query.SelectionSpec;
import org.hypertrace.core.documentstore.query.SortingSpec;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -85,7 +88,6 @@ void setUp() {

@AfterEach
void tearDown() {
verify(collection).getNamespace();
verifyNoMoreInteractions(collection, iterable, cursor, aggIterable);
}

Expand All @@ -98,6 +100,7 @@ public void testFindSimple() {
BasicDBObject mongoQuery = new BasicDBObject();
BasicDBObject projection = new BasicDBObject();

verify(collection).getNamespace();
verify(collection).find(mongoQuery);
verify(iterable).projection(projection);
verify(iterable, NOT_INVOKED).sort(any());
Expand All @@ -119,6 +122,7 @@ public void testFindWithSelection() {
BasicDBObject mongoQuery = new BasicDBObject();
BasicDBObject projection = BasicDBObject.parse("{id: 1, name: \"$fname\"}");

verify(collection).getNamespace();
verify(collection).find(mongoQuery);
verify(iterable).projection(projection);
verify(iterable, NOT_INVOKED).sort(any());
Expand Down Expand Up @@ -159,6 +163,7 @@ public void testFindWithFilter() {
+ "}");
BasicDBObject projection = new BasicDBObject();

verify(collection).getNamespace();
verify(collection).find(mongoQuery);
verify(iterable).projection(projection);
verify(iterable, NOT_INVOKED).sort(any());
Expand All @@ -181,6 +186,7 @@ public void testFindWithSorting() {
BasicDBObject sortQuery = BasicDBObject.parse("{ marks: -1, name: 1}");
BasicDBObject projection = new BasicDBObject();

verify(collection).getNamespace();
verify(collection).find(mongoQuery);
verify(iterable).projection(projection);
verify(iterable).sort(sortQuery);
Expand All @@ -199,6 +205,7 @@ public void testFindWithPagination() {
BasicDBObject mongoQuery = new BasicDBObject();
BasicDBObject projection = new BasicDBObject();

verify(collection).getNamespace();
verify(collection).find(mongoQuery);
verify(iterable).projection(projection);
verify(iterable, NOT_INVOKED).sort(any());
Expand Down Expand Up @@ -245,6 +252,7 @@ public void testFindWithAllClauses() {
BasicDBObject projection = BasicDBObject.parse("{id: 1, name: \"$fname\"}");
BasicDBObject sortQuery = BasicDBObject.parse("{ marks: -1, name: 1}");

verify(collection).getNamespace();
verify(collection).find(mongoQuery);
verify(iterable).projection(projection);
verify(iterable).sort(sortQuery);
Expand All @@ -253,6 +261,40 @@ public void testFindWithAllClauses() {
verify(iterable).cursor();
}

@Test
public void testFindAndAggregateWithDuplicateAlias() {
List<SelectionSpec> selectionSpecs =
List.of(
SelectionSpec.of(IdentifierExpression.of("item")),
SelectionSpec.of(IdentifierExpression.of("price"), "value"),
SelectionSpec.of(IdentifierExpression.of("quantity"), "value"),
SelectionSpec.of(IdentifierExpression.of("date")));
Selection selection = Selection.builder().selectionSpecs(selectionSpecs).build();
Filter filter =
Filter.builder()
.expression(
RelationalExpression.of(
IdentifierExpression.of("item"),
NOT_IN,
ConstantExpression.ofStrings(List.of("Soap", "Bottle"))))
.build();

Query query = Query.builder().setSelection(selection).setFilter(filter).build();

assertThrows(IllegalArgumentException.class, () -> executor.find(query));
verify(collection, NOT_INVOKED).getNamespace();
verify(collection, NOT_INVOKED).find(any(BasicDBObject.class));
verify(iterable, NOT_INVOKED).projection(any(BasicDBObject.class));
verify(iterable, NOT_INVOKED).sort(any(BasicDBObject.class));
verify(iterable, NOT_INVOKED).skip(anyInt());
verify(iterable, NOT_INVOKED).limit(anyInt());
verify(iterable, NOT_INVOKED).cursor();

assertThrows(IllegalArgumentException.class, () -> executor.aggregate(query));
verify(collection, NOT_INVOKED).aggregate(anyList());
verify(aggIterable, NOT_INVOKED).cursor();
}

@Test
public void testSimpleAggregate() {
Query query =
Expand Down Expand Up @@ -549,6 +591,7 @@ public void testGetDistinctCount() {

private void testAggregation(Query query, List<BasicDBObject> pipeline) {
executor.aggregate(query);
verify(collection).getNamespace();
verify(collection).aggregate(pipeline);
verify(aggIterable).cursor();
}
Expand Down

0 comments on commit 0b23b9a

Please sign in to comment.