Field type list pagination with querying and filtering.

graylog2-server/src/main/java/org/graylog2/database/filtering/DbFilterExpressionParser.java

@@ -18,9 +18,8 @@
 
 import com.mongodb.client.model.Filters;
 import org.bson.conversions.Bson;
+import org.graylog2.database.filtering.inmemory.SingleFilterParser;
 import org.graylog2.rest.resources.entities.EntityAttribute;
-import org.graylog2.search.SearchQueryField;
-import org.joda.time.DateTime;
 
 import java.util.List;
 import java.util.Map;
@@ -30,18 +29,15 @@
 
 public class DbFilterExpressionParser {
 
-    static final String FIELD_AND_VALUE_SEPARATOR = ":";
-    static final String RANGE_VALUES_SEPARATOR = "><";
-    static final String WRONG_FILTER_EXPR_FORMAT_ERROR_MSG =
-            "Wrong filter expression, <field_name>" + FIELD_AND_VALUE_SEPARATOR + "<field_value> format should be used";
+    private final SingleFilterParser singleFilterParser = new SingleFilterParser();
 
     public List<Bson> parse(final List<String> filterExpressions,
                             final List<EntityAttribute> attributes) {
         if (filterExpressions == null || filterExpressions.isEmpty()) {
             return List.of();
         }
         final Map<String, List<Filter>> groupedByField = filterExpressions.stream()
-                .map(expr -> parseSingleExpressionInner(expr, attributes))
+                .map(expr -> singleFilterParser.parseSingleExpression(expr, attributes))
                 .collect(groupingBy(Filter::field));
 
         return groupedByField.values().stream()
@@ -59,88 +55,10 @@ public List<Bson> parse(final List<String> filterExpressions,
     }
 
     public Bson parseSingleExpression(final String filterExpression, final List<EntityAttribute> attributes) {
-        final Filter filter = parseSingleExpressionInner(filterExpression, attributes);
+        final Filter filter = singleFilterParser.parseSingleExpression(filterExpression, attributes);
         return filter.toBson();
     }
 
-    private Filter parseSingleExpressionInner(final String filterExpression, final List<EntityAttribute> attributes) {
-        if (!filterExpression.contains(FIELD_AND_VALUE_SEPARATOR)) {
-            throw new IllegalArgumentException(WRONG_FILTER_EXPR_FORMAT_ERROR_MSG);
-        }
-        final String[] split = filterExpression.split(FIELD_AND_VALUE_SEPARATOR, 2);
-
-        final String fieldPart = split[0];
-        if (fieldPart == null || fieldPart.isEmpty()) {
-            throw new IllegalArgumentException(WRONG_FILTER_EXPR_FORMAT_ERROR_MSG);
-        }
-        final String valuePart = split[1];
-        if (valuePart == null || valuePart.isEmpty()) {
-            throw new IllegalArgumentException(WRONG_FILTER_EXPR_FORMAT_ERROR_MSG);
-        }
-
-        final EntityAttribute attributeMetaData = getAttributeMetaData(attributes, fieldPart);
-
-        final SearchQueryField.Type fieldType = attributeMetaData.type();
-        if (isRangeValueExpression(valuePart, fieldType)) {
-            if (valuePart.startsWith(RANGE_VALUES_SEPARATOR)) {
-                return new RangeFilter(attributeMetaData.id(),
-                        null,
-                        extractValue(fieldType, valuePart.substring(RANGE_VALUES_SEPARATOR.length()))
-                );
-            } else if (valuePart.endsWith(RANGE_VALUES_SEPARATOR)) {
-                return new RangeFilter(attributeMetaData.id(),
-                        extractValue(fieldType, valuePart.substring(0, valuePart.length() - RANGE_VALUES_SEPARATOR.length())),
-                        null
-                );
-            } else {
-                final String[] ranges = valuePart.split(RANGE_VALUES_SEPARATOR);
-                return new RangeFilter(attributeMetaData.id(),
-                        extractValue(fieldType, ranges[0]),
-                        extractValue(fieldType, ranges[1])
-                );
-            }
-        } else {
-            return new SingleValueFilter(attributeMetaData.id(), extractValue(fieldType, valuePart));
-        }
-
-    }
 
-    private Object extractValue(SearchQueryField.Type fieldType, String valuePart) {
-        final Object converted = fieldType.getMongoValueConverter().apply(valuePart);
-        if (converted instanceof DateTime && fieldType == SearchQueryField.Type.DATE) {
-            return ((DateTime) converted).toDate(); //MongoDB does not like Joda
-        } else {
-            return converted;
-        }
-
-    }
-
-    private EntityAttribute getAttributeMetaData(final List<EntityAttribute> attributes,
-                                                 final String attributeName) {
-        EntityAttribute matchingByTitle = null;
-
-        for (EntityAttribute attr : attributes) {
-            if (attributeName.equals(attr.id()) && isFilterable(attr)) {
-                return attr;
-            } else if (attributeName.equalsIgnoreCase(attr.title()) && isFilterable(attr)) {
-                matchingByTitle = attr;
-            }
-        }
-
-        if (matchingByTitle != null) {
-            return matchingByTitle;
-        } else {
-            throw new IllegalArgumentException(attributeName + " is not a field that can be used for filtering");
-        }
-
-    }
-
-    private boolean isRangeValueExpression(String valuePart, SearchQueryField.Type fieldType) {
-        return SearchQueryField.Type.NUMERIC_TYPES.contains(fieldType) && valuePart.contains(RANGE_VALUES_SEPARATOR);
-    }
-
-    private boolean isFilterable(final EntityAttribute attr) {
-        return Boolean.TRUE.equals(attr.filterable());
-    }
 
 }

graylog2-server/src/main/java/org/graylog2/database/filtering/Filter.java

@@ -17,9 +17,14 @@
 package org.graylog2.database.filtering;
 
 import org.bson.conversions.Bson;
+import org.graylog2.database.filtering.inmemory.InMemoryFilterable;
 
-interface Filter {
+import java.util.function.Predicate;
+
+public interface Filter {
     String field();
 
     Bson toBson();
+
+    Predicate<InMemoryFilterable> toPredicate();
 }

graylog2-server/src/main/java/org/graylog2/database/filtering/RangeFilter.java

@@ -19,11 +19,14 @@
 import com.mongodb.client.model.Filters;
 import org.bson.BsonDocument;
 import org.bson.conversions.Bson;
+import org.graylog2.database.filtering.inmemory.InMemoryFilterable;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.function.Predicate;
 
-record RangeFilter(String field, Object from, Object to) implements Filter {
+public record RangeFilter(String field, Object from, Object to) implements Filter {
 
     @Override
     public Bson toBson() {
@@ -40,4 +43,23 @@ public Bson toBson() {
             return new BsonDocument();
         }
     }
+
+    @Override
+    public Predicate<InMemoryFilterable> toPredicate() {
+        //TODO: we do not have a use case for that, but maybe this one needs to be implemented for future use cases...
+        throw new UnsupportedOperationException("RangeFilters are only supported in MongoDB-based filtering, not in in-memory filtering.");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final RangeFilter that = (RangeFilter) o;
+        return Objects.equals(field, that.field) && Objects.equals(from, that.from) && Objects.equals(to, that.to);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, from, to);
+    }
 }

graylog2-server/src/main/java/org/graylog2/database/filtering/SingleValueFilter.java

@@ -18,11 +18,35 @@
 
 import com.mongodb.client.model.Filters;
 import org.bson.conversions.Bson;
+import org.graylog2.database.filtering.inmemory.InMemoryFilterable;
 
-record SingleValueFilter(String field, Object value) implements Filter {
+import java.util.Objects;
+import java.util.function.Predicate;
+
+public record SingleValueFilter(String field, Object value) implements Filter {
 
     @Override
     public Bson toBson() {
         return Filters.eq(field(), value());
     }
+
+    @Override
+    public Predicate<InMemoryFilterable> toPredicate() {
+        return o -> o.extractFieldValue(field)
+                .map(fieldValue -> fieldValue.equals(value))
+                .orElse(false);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final SingleValueFilter that = (SingleValueFilter) o;
+        return Objects.equals(field, that.field) && Objects.equals(value, that.value);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, value);
+    }
 }

graylog2-server/src/main/java/org/graylog2/database/filtering/inmemory/InMemoryFilterExpressionParser.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.database.filtering.inmemory;
+
+import com.google.common.base.Predicates;
+import org.graylog2.database.filtering.Filter;
+import org.graylog2.rest.resources.entities.EntityAttribute;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.groupingBy;
+
+public class InMemoryFilterExpressionParser {
+
+    private final SingleFilterParser singleFilterParser = new SingleFilterParser();
+
+    public Predicate<InMemoryFilterable> parse(final List<String> filterExpressions,
+                                               final List<EntityAttribute> attributes) {
+        if (filterExpressions == null || filterExpressions.isEmpty()) {
+            return Predicates.alwaysTrue();
+        }
+        final Map<String, List<Filter>> groupedByField = filterExpressions.stream()
+                .map(expr -> singleFilterParser.parseSingleExpression(expr, attributes))
+                .collect(groupingBy(Filter::field));
+
+        return groupedByField.values().stream()
+                .map(grouped -> grouped.stream()
+                        .map(Filter::toPredicate)
+                        .collect(Collectors.toList()))
+                .map(groupedPredicates -> groupedPredicates.stream().reduce(Predicate::or).orElse(Predicates.alwaysTrue()))
+                .reduce(Predicate::and).orElse(Predicates.alwaysTrue());
+    }
+
+}

graylog2-server/src/main/java/org/graylog2/database/filtering/inmemory/InMemoryFilterable.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.database.filtering.inmemory;
+
+import java.util.Optional;
+
+public interface InMemoryFilterable {
+
+    Optional<Object> extractFieldValue(final String fieldName);
+}