From 496233da162a439a1e1cb6b63b6485bad7720c5a Mon Sep 17 00:00:00 2001 From: Zack King <91903901+kingzacko1@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:51:53 -0500 Subject: [PATCH] Add StreamCategoryFilter and stream_category to StreamDTO (#20110) * Add StreamCategoryFilter and stream_category to StreamDTO * Fix introduced test failures * Move StreamCategory resolution to SearchExecutor from SearchBackend * Add logic to populate queries with streamcategories with streamIds * Move streamcategory mapping from CommandFactory to MessagesResource * Replace StreamCategoryFilters with StreamFilters in place instead of at top level --- changelog/unreleased/pr-20110.toml | 5 + .../graylog/plugins/views/ViewsBindings.java | 2 + .../graylog/plugins/views/search/Query.java | 55 ++++++++- .../graylog/plugins/views/search/Search.java | 30 +++++ .../PluggableSearchNormalization.java | 19 +++- .../search/filter/StreamCategoryFilter.java | 104 +++++++++++++++++ .../views/search/rest/MessagesResource.java | 45 +++++--- .../org/graylog2/plugin/streams/Stream.java | 4 + .../java/org/graylog2/streams/StreamDTO.java | 13 ++- .../java/org/graylog2/streams/StreamImpl.java | 13 +++ .../org/graylog2/streams/StreamService.java | 2 + .../graylog2/streams/StreamServiceImpl.java | 11 ++ .../plugins/views/search/QueryTest.java | 107 ++++++++++++++++++ .../search/engine/SearchExecutorTest.java | 6 +- .../filter/StreamCategoryFilterTest.java | 77 +++++++++++++ .../search/rest/MessagesResourceTest.java | 2 +- .../rest/SearchResourceExecutionTest.java | 8 +- .../java/org/graylog2/streams/StreamMock.java | 12 ++ 18 files changed, 489 insertions(+), 26 deletions(-) create mode 100644 changelog/unreleased/pr-20110.toml create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java create mode 100644 graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java diff --git a/changelog/unreleased/pr-20110.toml b/changelog/unreleased/pr-20110.toml new file mode 100644 index 000000000000..f8c1a286322a --- /dev/null +++ b/changelog/unreleased/pr-20110.toml @@ -0,0 +1,5 @@ +type = "a" +message = "Added categories to Streams to allow Illuminate content to be scoped to multiple products." + +issues = ["graylog-plugin-enterprise#7945"] +pulls = ["20110"] diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java b/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java index f965568e40d7..b5f7862cefae 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java @@ -56,6 +56,7 @@ import org.graylog.plugins.views.search.filter.AndFilter; import org.graylog.plugins.views.search.filter.OrFilter; import org.graylog.plugins.views.search.filter.QueryStringFilter; +import org.graylog.plugins.views.search.filter.StreamCategoryFilter; import org.graylog.plugins.views.search.filter.StreamFilter; import org.graylog.plugins.views.search.querystrings.LastUsedQueryStringsService; import org.graylog.plugins.views.search.querystrings.MongoLastUsedQueryStringsService; @@ -177,6 +178,7 @@ protected void configure() { registerJacksonSubtype(AndFilter.class); registerJacksonSubtype(OrFilter.class); registerJacksonSubtype(StreamFilter.class); + registerJacksonSubtype(StreamCategoryFilter.class); registerJacksonSubtype(QueryStringFilter.class); // query backends for jackson diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java index 118157ea4523..7faeb0316e85 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java @@ -32,7 +32,9 @@ import org.graylog.plugins.views.search.engine.BackendQuery; import org.graylog.plugins.views.search.engine.EmptyTimeRange; import org.graylog.plugins.views.search.filter.AndFilter; +import org.graylog.plugins.views.search.filter.StreamCategoryFilter; import org.graylog.plugins.views.search.filter.StreamFilter; +import org.graylog.plugins.views.search.permissions.StreamPermissions; import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride; import org.graylog.plugins.views.search.rest.SearchTypeExecutionState; @@ -50,13 +52,17 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import static com.google.common.base.MoreObjects.firstNonNull; @@ -210,6 +216,20 @@ public Set usedStreamIds() { .orElse(Collections.emptySet()); } + @SuppressWarnings("UnstableApiUsage") + public Set usedStreamCategories() { + return Optional.ofNullable(filter()) + .map(optFilter -> { + final Traverser filterTraverser = Traverser.forTree(filter -> firstNonNull(filter.filters(), Collections.emptySet())); + return StreamSupport.stream(filterTraverser.breadthFirst(optFilter).spliterator(), false) + .filter(filter -> filter instanceof StreamCategoryFilter) + .map(streamFilter -> ((StreamCategoryFilter) streamFilter).category()) + .filter(Objects::nonNull) + .collect(toSet()); + }) + .orElse(Collections.emptySet()); + } + public Set streamIdsForPermissionsCheck() { final Set searchTypeStreamIds = searchTypes().stream() .map(SearchType::streams) @@ -219,7 +239,7 @@ public Set streamIdsForPermissionsCheck() { } public boolean hasStreams() { - return !usedStreamIds().isEmpty(); + return !(usedStreamIds().isEmpty() && usedStreamCategories().isEmpty()); } public boolean hasReferencedStreamFilters() { @@ -231,6 +251,39 @@ public Query addStreamsToFilter(Set streamIds) { return toBuilder().filter(newFilter).build(); } + public Query replaceStreamCategoryFilters(Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + if (filter() == null) { + return this; + } + return toBuilder() + .filter(streamCategoryToStreamFiltersRecursively(filter(), categoryMappingFunction, streamPermissions)) + .build(); + } + + private Filter streamCategoryToStreamFiltersRecursively(Filter filter, + Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + if (filter.filters() == null || filter.filters().isEmpty()) { + return filter; + } + Set mappedFilters = new HashSet<>(); + for (Filter f : filter.filters()) { + Filter mappedFilter = f; + if (f instanceof StreamCategoryFilter scf) { + mappedFilter = scf.toStreamFilter(categoryMappingFunction, streamPermissions); + } + if (mappedFilter != null) { + mappedFilter = streamCategoryToStreamFiltersRecursively(mappedFilter, categoryMappingFunction, streamPermissions); + mappedFilters.add(mappedFilter); + } + } + if (mappedFilters.isEmpty()) { + return null; + } + return filter.toGenericBuilder().filters(mappedFilters.stream().filter(Objects::nonNull).collect(toSet())).build(); + } + private Filter addStreamsTo(Filter filter, Set streamIds) { final Filter streamIdFilter = StreamFilter.anyIdOf(streamIds.toArray(new String[]{})); if (filter == null) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java index 22cf40f076f4..4eca94312a6a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java @@ -28,6 +28,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.graph.MutableGraph; +import org.graylog.plugins.views.search.permissions.StreamPermissions; import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.views.PluginMetadataSummary; import org.graylog2.contentpacks.ContentPackable; @@ -43,13 +44,17 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.stream.Collectors.toSet; @@ -146,10 +151,35 @@ public Search addStreamsToQueriesWithoutStreams(Supplier> defaultStr return toBuilder().queries(newQueries).build(); } + public Search addStreamsToQueriesWithCategories(Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + if (!hasQueriesWithStreamCategories()) { + return this; + } + final Set withStreamCategories = queries().stream().filter(q -> !q.usedStreamCategories().isEmpty()).collect(toSet()); + final Set withoutStreamCategories = Sets.difference(queries(), withStreamCategories); + final Set withMappedStreamCategories = new HashSet<>(); + + for (Query query : withStreamCategories) { + final Set mappedStreamIds = categoryMappingFunction.apply(query.usedStreamCategories()) + .filter(streamPermissions::canReadStream) + .collect(toSet()); + withMappedStreamCategories.add(query.addStreamsToFilter(mappedStreamIds)); + } + + final ImmutableSet newQueries = Sets.union(withMappedStreamCategories, withoutStreamCategories).immutableCopy(); + + return toBuilder().queries(newQueries).build(); + } + private boolean hasQueriesWithoutStreams() { return !queries().stream().allMatch(Query::hasStreams); } + private boolean hasQueriesWithStreamCategories() { + return queries().stream().anyMatch(q -> !q.usedStreamCategories().isEmpty()); + } + public abstract Builder toBuilder(); public static Builder builder() { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java index 0ca114b82ec7..d0d782acdd48 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java @@ -24,27 +24,34 @@ import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride; import org.graylog2.plugin.Tools; +import org.graylog2.streams.StreamService; import org.joda.time.DateTime; +import java.util.Collection; import java.util.Collections; import java.util.Optional; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; import static com.google.common.base.MoreObjects.firstNonNull; public class PluggableSearchNormalization implements SearchNormalization { private final Set pluggableNormalizers; private final Set postValidationNormalizers; + private final Function, Stream> streamCategoryMapper; @Inject public PluggableSearchNormalization(Set pluggableNormalizers, - @PostValidation Set postValidationNormalizers) { + @PostValidation Set postValidationNormalizers, + StreamService streamService) { this.pluggableNormalizers = pluggableNormalizers; this.postValidationNormalizers = postValidationNormalizers; + this.streamCategoryMapper = (categories) -> streamService.mapCategoriesToIds(categories).stream(); } - public PluggableSearchNormalization(Set pluggableNormalizers) { - this(pluggableNormalizers, Collections.emptySet()); + public PluggableSearchNormalization(Set pluggableNormalizers, StreamService streamService) { + this(pluggableNormalizers, Collections.emptySet(), streamService); } private Search normalize(Search search, Set normalizers) { @@ -68,7 +75,9 @@ private Query normalize(final Query query, @Override public Search preValidation(Search search, SearchUser searchUser, ExecutionState executionState) { - final Search searchWithStreams = search.addStreamsToQueriesWithoutStreams(() -> searchUser.streams().loadMessageStreamsWithFallback()); + final Search searchWithStreams = search + .addStreamsToQueriesWithoutStreams(() -> searchUser.streams().loadMessageStreamsWithFallback()) + .addStreamsToQueriesWithCategories(streamCategoryMapper, searchUser); final var now = referenceDateFromOverrideOrNow(executionState); final var normalizedSearch = searchWithStreams.applyExecutionState(firstNonNull(executionState, ExecutionState.empty())) .withReferenceDate(now); @@ -93,6 +102,8 @@ public Query preValidation(final Query query, final ParameterProvider parameterP Query normalizedQuery = query; if (!query.hasStreams()) { normalizedQuery = query.addStreamsToFilter(searchUser.streams().loadMessageStreamsWithFallback()); + } else if (!query.usedStreamCategories().isEmpty()) { + normalizedQuery = query.replaceStreamCategoryFilters(streamCategoryMapper, searchUser); } if (!executionState.equals(ExecutionState.empty())) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java new file mode 100644 index 000000000000..e3e37732d1e3 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.plugins.views.search.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.auto.value.AutoValue; +import org.graylog.plugins.views.search.Filter; +import org.graylog.plugins.views.search.permissions.StreamPermissions; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +@AutoValue +@JsonTypeName(StreamCategoryFilter.NAME) +@JsonDeserialize(builder = StreamCategoryFilter.Builder.class) +public abstract class StreamCategoryFilter implements Filter { + public static final String NAME = "stream_category"; + + @Override + @JsonProperty + public abstract String type(); + + @Override + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public abstract Set filters(); + + @JsonProperty("category") + public abstract String category(); + + public static Builder builder() { + return Builder.create(); + } + + public abstract Builder toBuilder(); + + public static StreamCategoryFilter ofCategory(String category) { + return builder().category(category).build(); + } + + public Filter toStreamFilter(Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + String[] mappedStreamIds = categoryMappingFunction.apply(List.of(category())) + .filter(streamPermissions::canReadStream) + .toArray(String[]::new); + // If the streamPermissions do not allow for any of the streams to be read, nullify this filter. + if (mappedStreamIds.length == 0) { + return null; + } + // Replace this category with an OrFilter of stream IDs and then add filters if they exist. + Filter streamFilter = StreamFilter.anyIdOf(mappedStreamIds).toGenericBuilder().build(); + if (filters() != null) { + streamFilter = streamFilter.toGenericBuilder().filters(filters()).build(); + } + return streamFilter; + } + + @Override + public Filter.Builder toGenericBuilder() { + return toBuilder(); + } + + @AutoValue.Builder + public abstract static class Builder implements Filter.Builder { + @JsonProperty + public abstract Builder type(String type); + + @JsonProperty + public abstract Builder filters(@Nullable Set filters); + + @JsonProperty("category") + public abstract Builder category(String category); + + public abstract StreamCategoryFilter build(); + + @JsonCreator + public static Builder create() { + return new AutoValue_StreamCategoryFilter.Builder().type(NAME); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java index 7547d5de6220..3ac00be40065 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java @@ -20,6 +20,16 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import jakarta.inject.Inject; +import jakarta.validation.Valid; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Context; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.glassfish.jersey.server.ChunkedOutput; import org.graylog.plugins.views.search.Search; @@ -49,25 +59,15 @@ import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.rest.MoreMediaTypes; import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; -import jakarta.inject.Inject; - -import jakarta.validation.Valid; - -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.Context; - +import java.util.Collection; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Stream; import static org.graylog2.shared.rest.documentation.generator.Generator.CLOUD_VISIBLE; @@ -81,6 +81,7 @@ public class MessagesResource extends RestResource implements PluginRestResource private final SearchExecutionGuard executionGuard; private final ExportJobService exportJobService; private final QueryValidationService queryValidationService; + private final Function, Stream> streamCategoryMapper; //allow mocking Function>, ChunkedOutput> asyncRunner = ChunkedRunner::runAsync; @@ -93,13 +94,27 @@ public MessagesResource( SearchDomain searchDomain, SearchExecutionGuard executionGuard, @SuppressWarnings("UnstableApiUsage") EventBus eventBus, - ExportJobService exportJobService, QueryValidationService queryValidationService) { + ExportJobService exportJobService, + QueryValidationService queryValidationService, + StreamService streamService) { + this(exporter, commandFactory, searchDomain, executionGuard, eventBus, exportJobService, queryValidationService, categories -> streamService.mapCategoriesToIds(categories).stream()); + } + + MessagesResource(MessagesExporter exporter, + CommandFactory commandFactory, + SearchDomain searchDomain, + SearchExecutionGuard executionGuard, + @SuppressWarnings("UnstableApiUsage") EventBus eventBus, + ExportJobService exportJobService, + QueryValidationService queryValidationService, + Function, Stream> streamCategoryMapper) { this.commandFactory = commandFactory; this.searchDomain = searchDomain; this.executionGuard = executionGuard; this.exportJobService = exportJobService; this.queryValidationService = queryValidationService; this.messagesExporterFactory = context -> new AuditingMessagesExporter(context, eventBus, exporter); + this.streamCategoryMapper = streamCategoryMapper; } @ApiOperation( @@ -246,7 +261,7 @@ private Search loadSearch(String searchId, ExecutionState executionState, Search .orElseThrow(() -> new NotFoundException("Search with id " + searchId + " does not exist")); search = search.addStreamsToQueriesWithoutStreams(() -> searchUser.streams().loadMessageStreamsWithFallback()); - + search = search.addStreamsToQueriesWithCategories(streamCategoryMapper, searchUser); search = search.applyExecutionState(executionState); executionGuard.check(search, searchUser::canReadStream); diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java b/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java index 39b52c3289b1..6ca9fad5c589 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java @@ -89,6 +89,8 @@ public static MatchingType valueOfOrDefault(String name) { String getContentPack(); + List getCategories(); + void setTitle(String title); void setDescription(String description); @@ -99,6 +101,8 @@ public static MatchingType valueOfOrDefault(String name) { void setMatchingType(MatchingType matchingType); + void setCategories(List categories); + Boolean isPaused(); Map asMap(List streamRules); diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java index c236a98dc5d1..78249f837d73 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java @@ -32,6 +32,7 @@ import javax.annotation.Nullable; import java.util.Collection; import java.util.Date; +import java.util.List; @AutoValue @WithBeanGetter @@ -54,6 +55,7 @@ public abstract class StreamDTO { public static final String FIELD_INDEX_SET_ID = "index_set_id"; public static final String EMBEDDED_ALERT_CONDITIONS = "alert_conditions"; public static final String FIELD_IS_EDITABLE = "is_editable"; + public static final String FIELD_CATEGORIES = "categories"; public static final Stream.MatchingType DEFAULT_MATCHING_TYPE = Stream.MatchingType.AND; @JsonProperty("id") @@ -114,6 +116,10 @@ public abstract class StreamDTO { @JsonProperty(FIELD_IS_EDITABLE) public abstract boolean isEditable(); + @JsonProperty(FIELD_CATEGORIES) + @Nullable + public abstract List categories(); + public abstract Builder toBuilder(); static Builder builder() { @@ -128,7 +134,8 @@ public static Builder create() { .matchingType(DEFAULT_MATCHING_TYPE.toString()) .isDefault(false) .isEditable(false) - .removeMatchesFromDefaultStream(false); + .removeMatchesFromDefaultStream(false) + .categories(List.of()); } @JsonProperty(FIELD_ID) @@ -181,6 +188,9 @@ public static Builder create() { @JsonProperty(FIELD_IS_EDITABLE) public abstract Builder isEditable(boolean isEditable); + @JsonProperty(FIELD_CATEGORIES) + public abstract Builder categories(List categories); + public abstract String id(); public abstract StreamDTO autoBuild(); @@ -206,6 +216,7 @@ public static StreamDTO fromDocument(Document document) { .creatorUserId(document.getString(FIELD_CREATOR_USER_ID)) .indexSetId(document.getString(FIELD_INDEX_SET_ID)) .outputs(document.getList(FIELD_OUTPUTS, ObjectId.class)) + .categories(document.getList(FIELD_CATEGORIES, String.class)) .build(); } } diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java index 61efaaacf3bd..345b2000cda0 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java @@ -63,6 +63,7 @@ public class StreamImpl extends PersistedImpl implements Stream { public static final String FIELD_DEFAULT_STREAM = "is_default_stream"; public static final String FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM = "remove_matches_from_default_stream"; public static final String FIELD_INDEX_SET_ID = "index_set_id"; + public static final String FIELD_CATEGORIES = "categories"; public static final String EMBEDDED_ALERT_CONDITIONS = "alert_conditions"; private final List streamRules; @@ -153,6 +154,17 @@ public void setContentPack(String contentPack) { fields.put(FIELD_CONTENT_PACK, contentPack); } + @Override + @SuppressWarnings("unchecked") + public List getCategories() { + return (List) fields.get(FIELD_CATEGORIES); + } + + @Override + public void setCategories(List categories) { + fields.put(FIELD_CATEGORIES, categories); + } + @Override public Boolean isPaused() { Boolean disabled = getDisabled(); @@ -189,6 +201,7 @@ public Map asMap() { result.put(FIELD_DEFAULT_STREAM, isDefaultStream()); result.put(FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM, getRemoveMatchesFromDefaultStream()); result.put(FIELD_INDEX_SET_ID, getIndexSetId()); + result.put(FIELD_CATEGORIES, getCategories()); return result; } diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java index a7121ae9ef6d..ff9cb13bbd0a 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java @@ -48,6 +48,8 @@ public interface StreamService extends PersistedService { Set loadByIds(Collection streamIds); + Set mapCategoriesToIds(Collection streamCategories); + Set indexSetIdsByIds(Collection streamIds); List loadAllEnabled(); diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java index 79a626959529..986fa601bb74 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java @@ -70,6 +70,7 @@ import static com.mongodb.client.model.Projections.excludeId; import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; +import static org.graylog2.streams.StreamImpl.FIELD_ID; import static org.graylog2.streams.StreamImpl.FIELD_INDEX_SET_ID; import static org.graylog2.streams.StreamImpl.FIELD_TITLE; @@ -282,6 +283,16 @@ public Set loadByIds(Collection streamIds) { return ImmutableSet.copyOf(loadAll(query)); } + @Override + public Set mapCategoriesToIds(Collection categories) { + final DBObject query = QueryBuilder.start(StreamImpl.FIELD_CATEGORIES).in(categories).get(); + final DBObject onlyIdField = DBProjection.include(FIELD_ID); + try (var cursor = collection(StreamImpl.class).find(query, onlyIdField); + var stream = StreamSupport.stream(cursor.spliterator(), false)) { + return stream.map(s -> s.get(FIELD_ID).toString()).collect(Collectors.toSet()); + } + } + @Override public Set indexSetIdsByIds(Collection streamIds) { Set dataStreamIds = streamIds.stream() diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java index e879b1347fa6..6134caac2b1d 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java @@ -34,6 +34,10 @@ import com.google.common.collect.ImmutableSet; import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString; import org.graylog.plugins.views.search.engine.BackendQuery; +import org.graylog.plugins.views.search.filter.AndFilter; +import org.graylog.plugins.views.search.filter.OrFilter; +import org.graylog.plugins.views.search.filter.QueryStringFilter; +import org.graylog.plugins.views.search.filter.StreamCategoryFilter; import org.graylog.plugins.views.search.filter.StreamFilter; import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride; @@ -62,9 +66,12 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class QueryTest { @@ -344,4 +351,104 @@ void appliesProperQueryExecutionStateIfEmptyGlobalOverride() { assertThat(query.query().queryString()) .isEqualTo("query"); } + + @Test + void replaceStreamCategoryFiltersWithStreamFilters() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("red", "blue", "yellow")); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("one", "two", "three")); + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.and(colorCategory, numberCategory)) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), streamId -> true); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isInstanceOf(AndFilter.class); + assertThat(filter.filters()).isNotNull(); + // The two StreamCategoryFilters should have been replaced with two OrFilters of three StreamFilters + assertThat(filter.filters()).hasSize(2); + assertThat(filter.filters().stream()).allSatisfy(f -> { + assertThat(f).isInstanceOf(OrFilter.class); + assertThat(f.filters()).isNotEmpty(); + assertThat(f.filters()).hasSize(3); + assertThat(f.filters().stream()).allSatisfy(f2 -> { + assertThat(f2).isInstanceOf(StreamFilter.class); + assertThat(f2.filters()).isNull(); + }); + }); + } + + @Test + void replaceStreamCategoryFiltersLeavesOtherFiltersAlone() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("red", "blue", "yellow")); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("one", "two", "three")); + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.builder() + .filters(ImmutableSet.builder() + .add(OrFilter.or(colorCategory, numberCategory)) + .add(QueryStringFilter.builder().query("source:localhost").build()) + .build()) + .build()) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), streamId -> true); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isInstanceOf(AndFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(2); + // The QueryStringFilter should have been left alone in the replacement + assertThat(filter.filters().stream()).satisfiesOnlyOnce(f -> { + assertThat(f).isInstanceOf(QueryStringFilter.class); + assertThat(f.filters()).isNull(); + assertThat(((QueryStringFilter) f).query()).isEqualTo("source:localhost"); + }); + } + + @Test + void replacementLeavesNoFilters() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(null); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(null); + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.and(colorCategory, numberCategory)) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), (streamId) -> false); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isNull(); + } + + @Test + void emptyReplacementFiltersAreRemoved() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(null); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(null); + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.builder() + .filters(ImmutableSet.builder() + .add(OrFilter.or(colorCategory, numberCategory)) + .add(QueryStringFilter.builder().query("source:localhost").build()) + .build()) + .build()) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), (streamId) -> false); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isInstanceOf(AndFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(1); + } } diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java index fe56cd4e7ec9..0f1a7c912449 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java @@ -42,6 +42,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.system.NodeId; import org.graylog2.shared.rest.exceptions.MissingStreamPermissionException; +import org.graylog2.streams.StreamService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -78,6 +79,9 @@ public class SearchExecutorTest { @Mock private NodeId nodeId; + @Mock + private StreamService streamService; + @Captor private ArgumentCaptor searchJobCaptor; @@ -97,7 +101,7 @@ void setUp() { Optional.of((queryString, job, query) -> PositionTrackingQuery.of("decorated")) ) ) - ))); + ), streamService)); when(queryEngine.execute(any(), any(), any())).thenAnswer(invocation -> { final SearchJob searchJob = invocation.getArgument(0); searchJob.addQueryResultFuture("query", CompletableFuture.completedFuture(QueryResult.emptyResult())); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java new file mode 100644 index 000000000000..6069a0521155 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.plugins.views.search.filter; + +import org.graylog.plugins.views.search.Filter; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StreamCategoryFilterTest { + + @Test + void testToStreamFilter() { + Filter filter = StreamCategoryFilter.ofCategory("colors"); + filter = ((StreamCategoryFilter) filter).toStreamFilter(this::categoryMapping, streamId -> true); + + assertThat(filter).isInstanceOf(OrFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(3); + assertThat(filter.filters().stream()).allSatisfy(f -> { + assertThat(f).isInstanceOf(StreamFilter.class); + assertThat(f.filters()).isNull(); + }); + } + + @Test + void testToStreamFilterWithPermissions() { + Filter filter = StreamCategoryFilter.ofCategory("colors"); + filter = ((StreamCategoryFilter) filter).toStreamFilter(this::categoryMapping, + (streamId) -> List.of("blue", "red", "one", "two").contains(streamId)); + + assertThat(filter).isInstanceOf(OrFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(2); + assertThat(filter.filters().stream()).allSatisfy(f -> { + assertThat(f).isInstanceOf(StreamFilter.class); + assertThat(f.filters()).isNull(); + assertThat(List.of("blue", "red")).contains(((StreamFilter)f).streamId()); + }); + } + + @Test + void testToStreamFilterReturnsNull() { + Filter filter = StreamCategoryFilter.ofCategory("colors"); + filter = ((StreamCategoryFilter) filter).toStreamFilter(this::categoryMapping, (streamId) -> false); + + assertThat(filter).isNull(); + } + + private Stream categoryMapping(Collection categories) { + Set streams = new HashSet<>(); + if (categories.contains("colors")) { + streams.addAll(List.of("red", "yellow", "blue")); + } + return streams.stream(); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java index 3e3709d9cae5..1c18ca59d143 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java @@ -91,7 +91,7 @@ void setUp() { class MessagesTestResource extends MessagesResource { public MessagesTestResource(MessagesExporter exporter, CommandFactory commandFactory, SearchDomain searchDomain, SearchExecutionGuard executionGuard, PermittedStreams permittedStreams, ObjectMapper objectMapper, EventBus eventBus, QueryValidationService validationService) { - super(exporter, commandFactory, searchDomain, executionGuard, eventBus, mock(ExportJobService.class), validationService); + super(exporter, commandFactory, searchDomain, executionGuard, eventBus, mock(ExportJobService.class), validationService, categories -> Stream.of()); } @Nullable diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java index 569f5435c074..dfdbf9c3327c 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java @@ -40,6 +40,7 @@ import org.graylog2.plugin.system.NodeId; import org.graylog2.plugin.system.SimpleNodeId; import org.graylog2.shared.rest.exceptions.MissingStreamPermissionException; +import org.graylog2.streams.StreamService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -58,9 +59,7 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -90,6 +89,9 @@ public class SearchResourceExecutionTest { @Mock private ClusterConfigService clusterConfigService; + @Mock + private StreamService streamService; + private final NodeId nodeId = new SimpleNodeId("5ca1ab1e-0000-4000-a000-000000000000"); private SearchResource searchResource; @@ -103,7 +105,7 @@ public void setUp() { searchJobService, queryEngine, new PluggableSearchValidation(executionGuard, Collections.emptySet()), - new PluggableSearchNormalization(Collections.emptySet())); + new PluggableSearchNormalization(Collections.emptySet(), streamService)); this.searchResource = new SearchResource(searchDomain, searchExecutor, searchJobService, eventBus, clusterConfigService) { @Override diff --git a/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java b/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java index 8515f4100cc8..f004f030c4eb 100644 --- a/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java +++ b/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java @@ -48,6 +48,7 @@ public class StreamMock implements Stream { private boolean disabled; private String contentPack; private List streamRules; + private List categories; private MatchingType matchingType; private boolean defaultStream; private boolean removeMatchesFromDefaultStream; @@ -69,6 +70,7 @@ public StreamMock(Map stream, List streamRules) { this.matchingType = (MatchingType) stream.getOrDefault(StreamImpl.FIELD_MATCHING_TYPE, MatchingType.AND); this.defaultStream = (boolean) stream.getOrDefault(StreamImpl.FIELD_DEFAULT_STREAM, false); this.removeMatchesFromDefaultStream = (boolean) stream.getOrDefault(StreamImpl.FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM, false); + this.categories = (List) stream.getOrDefault(StreamImpl.FIELD_CATEGORIES, List.of()); this.indexSet = new TestIndexSet(IndexSetConfig.create( "index-set-id", "title", @@ -134,6 +136,11 @@ public String getContentPack() { return contentPack; } + @Override + public List getCategories() { + return categories; + } + @Override public void setTitle(String title) { this.title = title; @@ -189,6 +196,11 @@ public void setMatchingType(MatchingType matchingType) { this.matchingType = matchingType; } + @Override + public void setCategories(List categories) { + this.categories = categories; + } + @Override public boolean isDefaultStream() { return defaultStream;