chore: adding the delete via filters operation for document store (#86)

* chore: adding the delete via filters operation for document store
hypertrace · Jun 29, 2022 · d40f191 · d40f191
1 parent f38193f
commit d40f191
Show file tree

Hide file tree

Showing 5 changed files with 215 additions and 26 deletions.
diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreTest.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreTest.java
@@ -216,6 +216,50 @@ public void testBulkUpsert(String dataStoreName) {
     assertEquals(5, collection.count());
   }
 
+  @ParameterizedTest
+  @MethodSource("databaseContextProvider")
+  public void testDeleteByDocFilter(String dataStoreName) {
+    Datastore datastore = datastoreMap.get(dataStoreName);
+    Collection collection = datastore.getCollection(COLLECTION_NAME);
+    Map<Key, Document> bulkMap = new HashMap<>();
+    bulkMap.put(new SingleValueKey("default", "testKey1"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey2"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey3"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey4"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey5"), Utils.createDocument("field", "value"));
+    bulkMap.put(
+        new SingleValueKey("default", "testKey6"),
+        Utils.createDocument("email", "[email protected]"));
+
+    assertTrue(collection.bulkUpsert(bulkMap));
+
+    collection.delete(org.hypertrace.core.documentstore.Filter.eq("field", "value"));
+    assertEquals(1, collection.count());
+  }
+
+  @ParameterizedTest
+  @MethodSource("databaseContextProvider")
+  public void testDeleteByFilterUnsupportedOperationException(String dataStoreName) {
+    Datastore datastore = datastoreMap.get(dataStoreName);
+    Collection collection = datastore.getCollection(COLLECTION_NAME);
+    Map<Key, Document> bulkMap = new HashMap<>();
+    bulkMap.put(new SingleValueKey("default", "testKey1"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey2"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey3"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey4"), Utils.createDocument("field", "value"));
+    bulkMap.put(new SingleValueKey("default", "testKey5"), Utils.createDocument("field", "value"));
+    bulkMap.put(
+        new SingleValueKey("default", "testKey6"),
+        Utils.createDocument("email", "[email protected]"));
+
+    assertTrue(collection.bulkUpsert(bulkMap));
+
+    UnsupportedOperationException exception =
+        assertThrows(UnsupportedOperationException.class, () -> collection.delete((Filter) null));
+    assertTrue(exception.getMessage().contains("Filter must be provided"));
+    assertEquals(6, collection.count());
+  }
+
   @ParameterizedTest
   @MethodSource("databaseContextProvider")
   public void testWithDifferentFieldTypes(String dataStoreName) throws Exception {
@@ -612,14 +656,18 @@ public void testSubDocumentUpdate(String dataStoreName) throws IOException {
     Assertions.assertFalse(documents.isEmpty());
 
     // mongo
-    // {"_lastUpdateTime":{"$date":"2021-03-14T18:53:14.914Z"},"createdTime":1615747994870,"foo1":"bar1","lastUpdatedTime":1615747994920,"subdoc":{"subfoo1":"subbar1","nesteddoc":{"nestedfoo1":"nestedbar1"}}}
+    // {"_lastUpdateTime":{"$date":"2021-03-14T18:53:14.914Z"},"createdTime":1615747994870,
+    // "foo1":"bar1","lastUpdatedTime":1615747994920,"subdoc":{"subfoo1":"subbar1",
+    // "nesteddoc":{"nestedfoo1":"nestedbar1"}}}
 
     // postgres
-    // {"foo1":"bar1","subdoc":{"subfoo1":"subbar1","nesteddoc":{"nestedfoo1":"nestedbar1"}},"created_at":"2021-03-15 00:24:50.981147","updated_at":"2021-03-15 00:24:50.981147"}
+    // {"foo1":"bar1","subdoc":{"subfoo1":"subbar1","nesteddoc":{"nestedfoo1":"nestedbar1"}},
+    // "created_at":"2021-03-15 00:24:50.981147","updated_at":"2021-03-15 00:24:50.981147"}
     System.out.println(documents.get(0).toJson());
     ObjectNode jsonNode = (ObjectNode) OBJECT_MAPPER.readTree(documents.get(0).toJson());
     String expected =
-        "{\"foo1\":\"bar1\",\"subdoc\":{\"subfoo1\":\"subbar1\",\"nesteddoc\":{\"nestedfoo1\":\"nestedbar1\"}}}";
+        "{\"foo1\":\"bar1\",\"subdoc\":{\"subfoo1\":\"subbar1\","
+            + "\"nesteddoc\":{\"nestedfoo1\":\"nestedbar1\"}}}";
     if (isMongo(dataStoreName)) {
       jsonNode.remove(MONGO_CREATED_TIME_KEY);
       jsonNode.remove(MONGO_LAST_UPDATE_TIME_KEY);
@@ -1044,7 +1092,13 @@ public void testSearchForNestedKey(String dataStoreName) throws IOException {
     Datastore datastore = datastoreMap.get(dataStoreName);
     Collection collection = datastore.getCollection(COLLECTION_NAME);
     String documentString =
-        "{\"attributes\":{\"trace_id\":{\"value\":{\"string\":\"00000000000000005e194fdf9fbf5101\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\",\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"tenantId\":\"__default\"}";
+        "{\"attributes\":{\"trace_id\":{\"value\":{\"string\":\"00000000000000005e194fdf9fbf5101"
+            + "\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},"
+            + "\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},"
+            + "\"FQN\":{\"value\":{\"string\":\"driver\"}}},"
+            + "\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\","
+            + "\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string"
+            + "\":\"driver\"}}},\"tenantId\":\"__default\"}";
     Document document = new JSONDocument(documentString);
     SingleValueKey key = new SingleValueKey("default", "testKey1");
     collection.upsert(key, document);
@@ -1067,19 +1121,42 @@ public void testSearch(String dataStoreName) throws IOException {
     Datastore datastore = datastoreMap.get(dataStoreName);
     Collection collection = datastore.getCollection(COLLECTION_NAME);
     String docStr1 =
-        "{\"amount\":1234.5,\"testKeyExist\":null,\"attributes\":{\"trace_id\":{\"value\":{\"string\":\"00000000000000005e194fdf9fbf5101\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\",\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"tenantId\":\"__default\"}";
+        "{\"amount\":1234.5,\"testKeyExist\":null,"
+            + "\"attributes\":{\"trace_id\":{\"value\":{\"string"
+            + "\":\"00000000000000005e194fdf9fbf5101\"}},"
+            + "\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},"
+            + "\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},"
+            + "\"FQN\":{\"value\":{\"string\":\"driver\"}}},"
+            + "\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\","
+            + "\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string"
+            + "\":\"driver\"}}},\"tenantId\":\"__default\"}";
     Document document1 = new JSONDocument(docStr1);
     SingleValueKey key1 = new SingleValueKey("default", "testKey1");
     collection.upsert(key1, document1);
 
     String docStr2 =
-        "{\"amount\":1234,\"testKeyExist\":123,\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":123,\"string\":\"00000000000000005e194fdf9fbf5101\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\",\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"tenantId\":\"__default\"}";
+        "{\"amount\":1234,\"testKeyExist\":123,"
+            + "\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":123,"
+            + "\"string\":\"00000000000000005e194fdf9fbf5101\"}},"
+            + "\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},"
+            + "\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},"
+            + "\"FQN\":{\"value\":{\"string\":\"driver\"}}},"
+            + "\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\","
+            + "\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string"
+            + "\":\"driver\"}}},\"tenantId\":\"__default\"}";
     Document document2 = new JSONDocument(docStr2);
     SingleValueKey key2 = new SingleValueKey("default", "testKey2");
     collection.upsert(key2, document2);
 
     String docStr3 =
-        "{\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":null,\"string\":\"00000000000000005e194fdf9fbf5101\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\",\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"tenantId\":\"__default\"}";
+        "{\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":null,"
+            + "\"string\":\"00000000000000005e194fdf9fbf5101\"}},"
+            + "\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},"
+            + "\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},"
+            + "\"FQN\":{\"value\":{\"string\":\"driver\"}}},"
+            + "\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\","
+            + "\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string"
+            + "\":\"driver\"}}},\"tenantId\":\"__default\"}";
     Document document3 = new JSONDocument(docStr3);
     SingleValueKey key3 = new SingleValueKey("default", "testKey3");
     collection.upsert(key3, document3);
@@ -1337,19 +1414,42 @@ public void testSearchIteratorInterface(String dataStoreName) throws IOException
     Datastore datastore = datastoreMap.get(dataStoreName);
     Collection collection = datastore.getCollection(COLLECTION_NAME);
     String docStr1 =
-        "{\"amount\":1234.5,\"testKeyExist\":null,\"attributes\":{\"trace_id\":{\"value\":{\"string\":\"00000000000000005e194fdf9fbf5101\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\",\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"tenantId\":\"__default\"}";
+        "{\"amount\":1234.5,\"testKeyExist\":null,"
+            + "\"attributes\":{\"trace_id\":{\"value\":{\"string"
+            + "\":\"00000000000000005e194fdf9fbf5101\"}},"
+            + "\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},"
+            + "\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},"
+            + "\"FQN\":{\"value\":{\"string\":\"driver\"}}},"
+            + "\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\","
+            + "\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string"
+            + "\":\"driver\"}}},\"tenantId\":\"__default\"}";
     Document document1 = new JSONDocument(docStr1);
     SingleValueKey key1 = new SingleValueKey("default", "testKey1");
     collection.upsert(key1, document1);
 
     String docStr2 =
-        "{\"amount\":1234,\"testKeyExist\":123,\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":123,\"string\":\"00000000000000005e194fdf9fbf5101\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\",\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"tenantId\":\"__default\"}";
+        "{\"amount\":1234,\"testKeyExist\":123,"
+            + "\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":123,"
+            + "\"string\":\"00000000000000005e194fdf9fbf5101\"}},"
+            + "\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},"
+            + "\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},"
+            + "\"FQN\":{\"value\":{\"string\":\"driver\"}}},"
+            + "\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\","
+            + "\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string"
+            + "\":\"driver\"}}},\"tenantId\":\"__default\"}";
     Document document2 = new JSONDocument(docStr2);
     SingleValueKey key2 = new SingleValueKey("default", "testKey2");
     collection.upsert(key2, document2);
 
     String docStr3 =
-        "{\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":null,\"string\":\"00000000000000005e194fdf9fbf5101\"}},\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\",\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string\":\"driver\"}}},\"tenantId\":\"__default\"}";
+        "{\"attributes\":{\"trace_id\":{\"value\":{\"testKeyExistNested\":null,"
+            + "\"string\":\"00000000000000005e194fdf9fbf5101\"}},"
+            + "\"span_id\":{\"value\":{\"string\":\"6449f1f720c93a67\"}},"
+            + "\"service_type\":{\"value\":{\"string\":\"JAEGER_SERVICE\"}},"
+            + "\"FQN\":{\"value\":{\"string\":\"driver\"}}},"
+            + "\"entityId\":\"e3ffc6f0-fc92-3a9c-9fa0-26269184d1aa\",\"entityName\":\"driver\","
+            + "\"entityType\":\"SERVICE\",\"identifyingAttributes\":{\"FQN\":{\"value\":{\"string"
+            + "\":\"driver\"}}},\"tenantId\":\"__default\"}";
     Document document3 = new JSONDocument(docStr3);
     SingleValueKey key3 = new SingleValueKey("default", "testKey3");
     collection.upsert(key3, document3);
@@ -1379,7 +1479,9 @@ public void testSearchIteratorInterface(String dataStoreName) throws IOException
       List<Document> documents = new ArrayList<>();
       while (true) {
         documents.add(results.next());
-        if (!results.hasNext()) break;
+        if (!results.hasNext()) {
+          break;
+        }
       }
       Assertions.assertEquals(1, documents.size());
     }
@@ -1425,10 +1527,9 @@ private Map<String, List<CreateUpdateTestThread>> executeCreateUpdateThreads(
   }
 
   /**
-   * mongo
-   * {"_lastUpdateTime":{"$date":"2021-03-14T15:43:04.842Z"},"createdTime":1615736584763,"foo1":"bar1","lastUpdatedTime":1615736584763}
-   * postgres {"foo1":"bar1","created_at":"2021-03-14 21:20:00.178909","updated_at":"2021-03-14
-   * 21:20:00.178909"}
+   * mongo {"_lastUpdateTime":{"$date":"2021-03-14T15:43:04.842Z"},"createdTime":1615736584763,
+   * "foo1":"bar1","lastUpdatedTime":1615736584763} postgres {"foo1":"bar1","created_at":"2021-03-14
+   * 21:20:00.178909","updated_at":"2021-03-14 21:20:00.178909"}
    */
   private static void verifyTimeRelatedFieldsPresent(String doc, String dataStoreName) {
     if (isMongo(dataStoreName)) {

diff --git a/...e/src/integrationTest/java/org/hypertrace/core/documentstore/mongo/MongoDocStoreTest.java b/...e/src/integrationTest/java/org/hypertrace/core/documentstore/mongo/MongoDocStoreTest.java
@@ -7,6 +7,7 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -300,6 +301,39 @@ public void testSelectAll() throws IOException {
     assertEquals(1, collection.count());
   }
 
+  @Test
+  public void testDeleteByFilter() throws IOException {
+    datastore.createCollection(COLLECTION_NAME, null);
+    Collection collection = datastore.getCollection(COLLECTION_NAME);
+    collection.upsert(
+        new SingleValueKey("default", "testKey1"), Utils.createDocument("field", "value"));
+    collection.upsert(
+        new SingleValueKey("default", "testKey2"), Utils.createDocument("field", "value"));
+    collection.upsert(
+        new SingleValueKey("default", "testKey3"), Utils.createDocument("field", "value1"));
+    assertEquals(3, collection.count());
+    // Delete one of the documents and test again.
+    collection.delete(org.hypertrace.core.documentstore.Filter.eq("field", "value"));
+    assertEquals(1, collection.count());
+  }
+
+  @Test
+  public void testDeleteByFilterUnsupportedOperation() throws IOException {
+    datastore.createCollection(COLLECTION_NAME, null);
+    Collection collection = datastore.getCollection(COLLECTION_NAME);
+    collection.upsert(
+        new SingleValueKey("default", "testKey1"), Utils.createDocument("field", "value"));
+    collection.upsert(
+        new SingleValueKey("default", "testKey2"), Utils.createDocument("field", "value"));
+    collection.upsert(
+        new SingleValueKey("default", "testKey3"), Utils.createDocument("field", "value1"));
+    assertEquals(3, collection.count());
+    UnsupportedOperationException exception =
+        assertThrows(UnsupportedOperationException.class, () -> collection.delete((Filter) null));
+    assertTrue(exception.getMessage().contains("Filter must be provided"));
+    assertEquals(3, collection.count());
+  }
+
   @Test
   public void testSelections() throws IOException {
     datastore.createCollection(COLLECTION_NAME, null);
@@ -428,9 +462,9 @@ public void testReturnAndBulkUpsert() throws IOException {
     Map<Key, Document> documentMapV1 =
         Map.of(
             new SingleValueKey("default", "testKey1"),
-                Utils.createDocument("id", "1", "testKey1", "abc-v1"),
+            Utils.createDocument("id", "1", "testKey1", "abc-v1"),
             new SingleValueKey("default", "testKey2"),
-                Utils.createDocument("id", "2", "testKey2", "xyz-v1"));
+            Utils.createDocument("id", "2", "testKey2", "xyz-v1"));
 
     Iterator<Document> iterator = collection.bulkUpsertAndReturnOlderDocuments(documentMapV1);
     // Initially there shouldn't be any documents.
@@ -440,9 +474,9 @@ public void testReturnAndBulkUpsert() throws IOException {
     Map<Key, Document> documentMapV2 =
         Map.of(
             new SingleValueKey("default", "testKey1"),
-                Utils.createDocument("id", "1", "testKey1", "abc-v2"),
+            Utils.createDocument("id", "1", "testKey1", "abc-v2"),
             new SingleValueKey("default", "testKey2"),
-                Utils.createDocument("id", "2", "testKey2", "xyz-v2"));
+            Utils.createDocument("id", "2", "testKey2", "xyz-v2"));
     iterator = collection.bulkUpsertAndReturnOlderDocuments(documentMapV2);
     assertEquals(2, collection.count());
     List<Document> documents = new ArrayList<>();

diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/Collection.java b/document-store/src/main/java/org/hypertrace/core/documentstore/Collection.java
@@ -88,6 +88,14 @@ public interface Collection {
    */
   boolean delete(Key key);
 
+  /**
+   * Delete the document matching the given filter.
+   *
+   * @param filter The filter to determine documents to be deleted. Only the filter clause.
+   * @return True if the documents are deleted, false otherwise.
+   */
+  boolean delete(Filter filter);
+
   /**
    * Delete the documents for the given keys
    *