Dw/async exec via search apis

graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java

@@ -28,6 +28,7 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.graph.MutableGraph;
+import org.graylog.plugins.views.search.engine.validation.DataWarehouseSearchValidator;
 import org.graylog.plugins.views.search.permissions.StreamPermissions;
 import org.graylog.plugins.views.search.rest.ExecutionState;
 import org.graylog.plugins.views.search.views.PluginMetadataSummary;
@@ -133,7 +134,7 @@ public Search applyExecutionState(final ExecutionState executionState) {
 
 
     public Search addStreamsToQueriesWithoutStreams(Supplier<Set<String>> defaultStreamsSupplier) {
-        if (!hasQueriesWithoutStreams()) {
+        if (!hasQueriesWithoutStreams() || DataWarehouseSearchValidator.containsDataWarehouseSearchElements(this)) {
             return this;
         }
         final Set<Query> withStreams = queries().stream().filter(Query::hasStreams).collect(toSet());
@@ -157,7 +158,7 @@ public Search addStreamsToQueriesWithoutStreams(Supplier<Set<String>> defaultStr
 
     public Search addStreamsToQueriesWithCategories(Function<Collection<String>, Stream<String>> categoryMappingFunction,
                                                     StreamPermissions streamPermissions) {
-        if (!hasQueriesWithStreamCategories()) {
+        if (!hasQueriesWithStreamCategories() || DataWarehouseSearchValidator.containsDataWarehouseSearchElements(this)) {
             return this;
         }
         final Set<Query> withStreamCategories = queries().stream().filter(q -> !q.usedStreamCategories().isEmpty()).collect(toSet());
@@ -178,7 +179,7 @@ public Search addStreamsToQueriesWithCategories(Function<Collection<String>, Str
 
     public Search addStreamsToSearchTypesWithCategories(Function<Collection<String>, Stream<String>> categoryMappingFunction,
                                                         StreamPermissions streamPermissions) {
-        if (!hasQuerySearchTypesWithStreamCategories()) {
+        if (!hasQuerySearchTypesWithStreamCategories() || DataWarehouseSearchValidator.containsDataWarehouseSearchElements(this)) {
             return this;
         }
         final Set<Query> withStreamCategories = queries().stream()

graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchType.java

@@ -25,6 +25,7 @@
 import org.graylog.plugins.views.search.engine.BackendQuery;
 import org.graylog.plugins.views.search.rest.SearchTypeExecutionState;
 import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+import org.graylog.plugins.views.search.searchtypes.SearchEngineSearchType;
 import org.graylog.plugins.views.search.timeranges.DerivedTimeRange;
 import org.graylog2.contentpacks.ContentPackable;
 import org.graylog2.contentpacks.EntityDescriptorIds;
@@ -140,7 +141,7 @@ interface Result {
     }
 
     @JsonAutoDetect
-    class Fallback implements SearchType {
+    class Fallback implements SearchEngineSearchType {
 
         @JsonProperty
         private String type;

graylog2-server/src/main/java/org/graylog/plugins/views/search/elasticsearch/ElasticsearchQueryString.java

@@ -23,13 +23,14 @@
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.auto.value.AutoValue;
 import com.google.common.base.Strings;
+import jakarta.validation.constraints.NotNull;
+import org.graylog.plugins.views.search.SearchType;
 import org.graylog.plugins.views.search.engine.BackendQuery;
+import org.graylog.plugins.views.search.searchtypes.SearchEngineSearchType;
 
 import javax.annotation.Nullable;
-
-import jakarta.validation.constraints.NotNull;
-
 import java.util.Optional;
+import java.util.Set;
 
 @AutoValue
 @JsonAutoDetect
@@ -65,6 +66,11 @@ public static ElasticsearchQueryString create(final @JsonProperty("type") String
     @Override
     public abstract String queryString();
 
+    @JsonIgnore
+    public Set<Class<? extends SearchType>> supportedSearchTypes() {
+        return Set.of(SearchEngineSearchType.class);
+    }
+
     @JsonIgnore
     public boolean isEmpty() {
         String trimmed = queryString().trim();

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/BackendQuery.java

@@ -16,9 +16,13 @@
  */
 package org.graylog.plugins.views.search.engine;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.graylog.plugins.views.search.SearchType;
 import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString;
 
+import java.util.Set;
+
 @JsonTypeInfo(
         use = JsonTypeInfo.Id.NAME,
         property = BackendQuery.TYPE_FIELD,
@@ -30,4 +34,7 @@ public interface BackendQuery {
     String type();
 
     String queryString();
+
+    @JsonIgnore
+    Set<Class<? extends SearchType>> supportedSearchTypes();
 }

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/EngineBindings.java

@@ -20,7 +20,9 @@
 import org.graylog.plugins.views.search.engine.normalization.DecorateQueryStringsNormalizer;
 import org.graylog.plugins.views.search.engine.normalization.PluggableSearchNormalization;
 import org.graylog.plugins.views.search.engine.normalization.SearchNormalization;
+import org.graylog.plugins.views.search.engine.validation.DataWarehouseSearchValidator;
 import org.graylog.plugins.views.search.engine.validation.PluggableSearchValidation;
+import org.graylog.plugins.views.search.engine.validation.SearchTypesMatchBackendQueryValidator;
 import org.graylog.plugins.views.search.engine.validation.SearchValidation;
 import org.graylog.plugins.views.search.engine.validation.TimeRangeValidator;
 
@@ -34,5 +36,8 @@ protected void configure() {
         // Triggering set binder explicitly, so no injection errors are being caused if no implementation is bound.
         searchPostValidationNormalizerBinder();
         registerSearchValidator(TimeRangeValidator.class);
+        registerSearchValidator(SearchTypesMatchBackendQueryValidator.class);
+        //while DW searches are enterprise only, the validation should probably be present in gl-server, i.e. to discover invalid searches even if a client has lost the license
+        registerSearchValidator(DataWarehouseSearchValidator.class);
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/validation/DataWarehouseSearchValidator.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.engine.validation;
+
+import com.google.common.collect.ImmutableSet;
+import org.graylog.plugins.views.search.Query;
+import org.graylog.plugins.views.search.Search;
+import org.graylog.plugins.views.search.SearchType;
+import org.graylog.plugins.views.search.errors.QueryError;
+import org.graylog.plugins.views.search.errors.SearchError;
+import org.graylog.plugins.views.search.errors.SearchTypeError;
+import org.graylog.plugins.views.search.permissions.SearchUser;
+import org.graylog.plugins.views.search.searchtypes.DataWarehouseSearchType;
+
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class DataWarehouseSearchValidator implements SearchValidator {
+
+    @Override
+    public Set<SearchError> validate(final Search search,
+                                     final SearchUser searchUser) {
+        //this should be either validated elsewhere or impossible
+        assert search.queries() != null;
+        assert !search.queries().isEmpty();
+
+        if (containsDataWarehouseSearchElements(search)) {
+            if (search.queries().size() > 1) {
+                return wholeSearchInvalid(search, "Data Warehouse elements present in Search, only 1 query allowed for those type of searches");
+            }
+            return validate(search.queries().stream().findFirst().get(), searchUser);
+        } else {
+            return Set.of();
+        }
+    }
+
+    @Override
+    public Set<SearchError> validate(final Query query,
+                                     final SearchUser searchUser) {
+        if (containsDataWarehouseSearchElements(query)) {
+            final ImmutableSet<SearchType> searchTypes = query.searchTypes();
+            if (searchTypes.size() != 1) {
+                return Set.of(new QueryError(query, "Data Warehouse query can contain only one search type"));
+            }
+            final Optional<SearchType> first = searchTypes.stream().findFirst();
+
+            final Set<String> streams = first.get().streams();
+            if (streams == null || streams.size() > 1) {
+                return Set.of(new SearchTypeError(query, first.get().id(),
+                        "Data Warehouse preview can be executed on only 1 stream, search type contained more"));
+            }
+
+        }
+        return Set.of();
+    }
+
+    public static boolean containsDataWarehouseSearchElements(final Search search) {
+        return search.queries().stream().anyMatch(DataWarehouseSearchValidator::containsDataWarehouseSearchElements);
+    }
+
+    public static boolean containsDataWarehouseSearchElements(final Query query) {
+        return (query.query().type().startsWith(DataWarehouseSearchType.PREFIX))
+                || (query.searchTypes().stream().anyMatch(searchType -> searchType instanceof DataWarehouseSearchType));
+    }
+
+    private Set<SearchError> wholeSearchInvalid(final Search search, final String explanation) {
+        return search.queries()
+                .stream()
+                .map(query -> new QueryError(query, explanation))
+                .collect(Collectors.toSet());
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/validation/SearchTypesMatchBackendQueryValidator.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.engine.validation;
+
+import org.graylog.plugins.views.search.Query;
+import org.graylog.plugins.views.search.Search;
+import org.graylog.plugins.views.search.errors.SearchError;
+import org.graylog.plugins.views.search.errors.SearchTypeError;
+import org.graylog.plugins.views.search.permissions.SearchUser;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class SearchTypesMatchBackendQueryValidator implements SearchValidator {
+
+    public static final String INVALID_SEARCH_TYPE_FOR_GIVEN_QUERY_TYPE_MSG = "Invalid search type for given query type";
+
+    @Override
+    public Set<SearchError> validate(final Search search,
+                                     final SearchUser searchUser) {
+        return search.queries()
+                .stream()
+                .flatMap(query -> validate(query, searchUser).stream())
+                .collect(Collectors.toSet());
+    }
+
+    @Override
+    public Set<SearchError> validate(final Query query,
+                                     final SearchUser searchUser) {
+        return query.searchTypes().stream()
+                .filter(searchType -> query.query()
+                        .supportedSearchTypes()
+                        .stream()
+                        .anyMatch(supportedType -> !supportedType.isAssignableFrom(searchType.getClass())))
+                .map(searchType -> new SearchTypeError(query, searchType.id(), INVALID_SEARCH_TYPE_FOR_GIVEN_QUERY_TYPE_MSG))
+                .collect(Collectors.toSet());
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/DataWarehouseSearchType.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.searchtypes;
+
+import org.graylog.plugins.views.search.SearchType;
+
+/**
+ * Marker interface for search types that are not search engine related, but Data Warehouse/Iceberg related
+ */
+public interface DataWarehouseSearchType extends SearchType {
+
+    String PREFIX = "data_warehouse_";
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/MessageList.java

@@ -52,7 +52,7 @@
 @AutoValue
 @JsonTypeName(MessageList.NAME)
 @JsonDeserialize(builder = MessageList.Builder.class)
-public abstract class MessageList implements SearchType {
+public abstract class MessageList implements SearchEngineSearchType {
     public static final String NAME = "messages";
 
     @Override

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/SearchEngineSearchType.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.searchtypes;
+
+import org.graylog.plugins.views.search.SearchType;
+
+/**
+ * Marker interface for search types that are search engine related.
+ */
+public interface SearchEngineSearchType extends SearchType {
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/events/EventList.java

@@ -29,6 +29,7 @@
 import org.graylog.plugins.views.search.engine.BackendQuery;
 import org.graylog.plugins.views.search.rest.SearchTypeExecutionState;
 import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+import org.graylog.plugins.views.search.searchtypes.SearchEngineSearchType;
 import org.graylog.plugins.views.search.timeranges.DerivedTimeRange;
 import org.graylog2.contentpacks.EntityDescriptorIds;
 import org.graylog2.contentpacks.model.entities.EntityDescriptor;
@@ -52,7 +53,7 @@
 @AutoValue
 @JsonTypeName(EventList.NAME)
 @JsonDeserialize(builder = EventList.Builder.class)
-public abstract class EventList implements SearchType, HasAttributeFilter {
+public abstract class EventList implements SearchEngineSearchType, HasAttributeFilter {
     public static final int DEFAULT_PAGE_SIZE = 10;
     public static final String NAME = "events";
     public static final Set<String> KNOWN_ATTRIBUTES = Set.of("priority", "event_definition_id", "alert");

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/pivot/Pivot.java

@@ -28,6 +28,7 @@
 import org.graylog.plugins.views.search.engine.BackendQuery;
 import org.graylog.plugins.views.search.rest.SearchTypeExecutionState;
 import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+import org.graylog.plugins.views.search.searchtypes.SearchEngineSearchType;
 import org.graylog.plugins.views.search.timeranges.DerivedTimeRange;
 import org.graylog2.contentpacks.EntityDescriptorIds;
 import org.graylog2.contentpacks.model.entities.EntityDescriptor;
@@ -47,7 +48,7 @@
 @AutoValue
 @JsonTypeName(Pivot.NAME)
 @JsonDeserialize(builder = Pivot.Builder.class)
-public abstract class Pivot implements SearchType {
+public abstract class Pivot implements SearchEngineSearchType {
     public static final String NAME = "pivot";
 
     @Override