diff --git a/changelog/unreleased/issue-16885.toml b/changelog/unreleased/issue-16885.toml new file mode 100644 index 000000000000..13c5e4b678b1 --- /dev/null +++ b/changelog/unreleased/issue-16885.toml @@ -0,0 +1,19 @@ +type = "changed" +message = "Changed Content Packs handling to allow import/export of entites that reference Streams by title." + +pulls = ["16743"] + +details.user = """ +Previously it was not possible to create a Content Pack with Stream scoped entities without also exporting the referenced Stream, +and potentially duplicating the Stream. + +For example, if a user had a Dashboard with a widget that was scoped to "stream_xyz" that they wished to create a +Content Pack with to use on another system, there were two options: +- Remove the Stream from the Dashboard widget before export, then re-associate the Stream with the Dashboard after uploading. +- Export the Stream along with the Dashboard, in which case a new "stream_xyz" would be created on the uploading system + (whether it already existed or not). + +This change allows users to create a Content Pack with a "stream_xyz" scoped Dashboard, referencing the Stream by title only. +When uploaded and installed, the Content Pack will resolve the existing stream with title "stream_xyz", +and associate it to the new Dashboard. +""" diff --git a/graylog2-server/src/main/java/org/graylog/events/contentpack/entities/AggregationEventProcessorConfigEntity.java b/graylog2-server/src/main/java/org/graylog/events/contentpack/entities/AggregationEventProcessorConfigEntity.java index f94b4d5e3fb0..1461740c3af4 100644 --- a/graylog2-server/src/main/java/org/graylog/events/contentpack/entities/AggregationEventProcessorConfigEntity.java +++ b/graylog2-server/src/main/java/org/graylog/events/contentpack/entities/AggregationEventProcessorConfigEntity.java @@ -27,8 +27,6 @@ import org.graylog.events.processor.aggregation.AggregationConditions; import org.graylog.events.processor.aggregation.AggregationEventProcessorConfig; import org.graylog2.contentpacks.exceptions.ContentPackException; -import org.graylog2.contentpacks.model.ModelId; -import org.graylog2.contentpacks.model.ModelTypes; import org.graylog2.contentpacks.model.entities.Entity; import org.graylog2.contentpacks.model.entities.EntityDescriptor; import org.graylog2.contentpacks.model.entities.EntityV1; @@ -42,6 +40,9 @@ import java.util.Optional; import java.util.stream.Collectors; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntity; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; + @AutoValue @JsonTypeName(AggregationEventProcessorConfigEntity.TYPE_NAME) @JsonDeserialize(builder = AggregationEventProcessorConfigEntity.Builder.class) @@ -129,8 +130,7 @@ public EventProcessorConfig toNativeEntity(Map parameter Map nativeEntities) { final ImmutableSet streamSet = ImmutableSet.copyOf( streams().stream() - .map(id -> EntityDescriptor.create(id, ModelTypes.STREAM_V1)) - .map(nativeEntities::get) + .map(id -> resolveStreamEntityObject(id, nativeEntities)) .map(object -> { if (object == null) { throw new ContentPackException("Missing Stream for event definition"); @@ -162,9 +162,7 @@ public void resolveForInstallation(EntityV1 entity, Map entities, MutableGraph graph) { streams().stream() - .map(ModelId::of) - .map(modelId -> EntityDescriptor.create(modelId, ModelTypes.STREAM_V1)) - .map(entities::get) + .map(id -> resolveStreamEntity(id, entities)) .filter(Objects::nonNull) .forEach(stream -> graph.putEdge(entity, stream)); } diff --git a/graylog2-server/src/main/java/org/graylog/events/processor/aggregation/AggregationEventProcessorConfig.java b/graylog2-server/src/main/java/org/graylog/events/processor/aggregation/AggregationEventProcessorConfig.java index f2cf15386a30..31a235a39090 100644 --- a/graylog2-server/src/main/java/org/graylog/events/processor/aggregation/AggregationEventProcessorConfig.java +++ b/graylog2-server/src/main/java/org/graylog/events/processor/aggregation/AggregationEventProcessorConfig.java @@ -53,6 +53,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityId; import static org.graylog2.shared.utilities.StringUtils.f; @AutoValue @@ -247,7 +248,7 @@ private void checkEventLimitGreaterZero(ValidationResult validationResult) { @Override public EventProcessorConfigEntity toContentPackEntity(EntityDescriptorIds entityDescriptorIds) { final ImmutableSet streamRefs = ImmutableSet.copyOf(streams().stream() - .map(streamId -> entityDescriptorIds.get(streamId, ModelTypes.STREAM_V1)) + .map(streamId -> getStreamEntityId(streamId, entityDescriptorIds)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toSet())); @@ -269,7 +270,7 @@ public void resolveNativeEntity(EntityDescriptor entityDescriptor, MutableGraph< streams().forEach(streamId -> { final EntityDescriptor depStream = EntityDescriptor.builder() .id(ModelId.of(streamId)) - .type(ModelTypes.STREAM_V1) + .type(ModelTypes.STREAM_REF_V1) .build(); mutableGraph.putEdge(entityDescriptor, depStream); }); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java index cf4262d9a3fd..39f362c531f7 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java @@ -40,7 +40,6 @@ import org.graylog.plugins.views.search.searchfilters.model.UsesSearchFilters; import org.graylog2.contentpacks.ContentPackable; import org.graylog2.contentpacks.EntityDescriptorIds; -import org.graylog2.contentpacks.model.ModelTypes; import org.graylog2.contentpacks.model.entities.QueryEntity; import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; @@ -60,6 +59,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableSortedSet.of; import static java.util.stream.Collectors.toSet; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityIdOrThrow; @AutoValue @JsonAutoDetect @@ -290,8 +290,7 @@ private Filter shallowMappedFilter(EntityDescriptorIds entityDescriptorIds) { .map(filter -> { if (filter.type().equals(StreamFilter.NAME)) { final StreamFilter streamFilter = (StreamFilter) filter; - final String streamId = entityDescriptorIds. - getOrThrow(streamFilter.streamId(), ModelTypes.STREAM_V1); + final String streamId = getStreamEntityIdOrThrow(streamFilter.streamId(), entityDescriptorIds); return streamFilter.toBuilder().streamId(streamId).build(); } return filter; diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchType.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchType.java index 54fcd7b6c05f..15376f7e6ae8 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchType.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchType.java @@ -29,8 +29,6 @@ import org.graylog2.contentpacks.ContentPackable; import org.graylog2.contentpacks.EntityDescriptorIds; import org.graylog2.contentpacks.exceptions.ContentPackException; -import org.graylog2.contentpacks.model.ModelTypes; -import org.graylog2.contentpacks.model.entities.EntityDescriptor; import org.graylog2.contentpacks.model.entities.SearchTypeEntity; import javax.annotation.Nullable; @@ -42,6 +40,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityId; + /** * A search type represents parts of a query that generates a {@see Result result}. *

@@ -243,7 +243,7 @@ public SearchTypeEntity toContentPackEntity(EntityDescriptorIds entityDescriptor default Set mappedStreams(EntityDescriptorIds entityDescriptorIds) { return streams().stream() - .map(streamId -> entityDescriptorIds.get(EntityDescriptor.create(streamId, ModelTypes.STREAM_V1))) + .map(streamId -> getStreamEntityId(streamId, entityDescriptorIds)) .map(optionalStreamId -> optionalStreamId.orElseThrow(() -> new ContentPackException("Did not find matching stream id"))) .collect(Collectors.toSet()); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/WidgetDTO.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/WidgetDTO.java index db54ae19417a..14871696e541 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/WidgetDTO.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/WidgetDTO.java @@ -27,8 +27,6 @@ import org.graylog.plugins.views.search.searchfilters.model.UsesSearchFilters; import org.graylog2.contentpacks.ContentPackable; import org.graylog2.contentpacks.EntityDescriptorIds; -import org.graylog2.contentpacks.model.ModelTypes; -import org.graylog2.contentpacks.model.entities.EntityDescriptor; import org.graylog2.contentpacks.model.entities.WidgetEntity; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; @@ -39,6 +37,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityId; + @AutoValue @JsonDeserialize(builder = WidgetDTO.Builder.class) @WithBeanGetter @@ -128,8 +128,8 @@ static Builder builder() { @Override public WidgetEntity toContentPackEntity(EntityDescriptorIds entityDescriptorIds) { - Set mappedStreams = streams().stream().map(streamId -> - entityDescriptorIds.get(EntityDescriptor.create(streamId, ModelTypes.STREAM_V1))) + Set mappedStreams = streams().stream() + .map(streamId -> getStreamEntityId(streamId, entityDescriptorIds)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toSet()); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java index 14c573021fba..b5451f95b7df 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java @@ -32,6 +32,7 @@ import org.graylog2.contentpacks.facades.SidecarCollectorConfigurationFacade; import org.graylog2.contentpacks.facades.SidecarCollectorFacade; import org.graylog2.contentpacks.facades.StreamFacade; +import org.graylog2.contentpacks.facades.StreamReferenceFacade; import org.graylog2.contentpacks.facades.UrlWhitelistFacade; import org.graylog2.contentpacks.facades.dashboardV1.DashboardV1Facade; import org.graylog2.contentpacks.jersey.ModelIdParamConverter; @@ -61,6 +62,7 @@ protected void configure() { addEntityFacade(PipelineRuleFacade.TYPE_V1, PipelineRuleFacade.class); addEntityFacade(RootEntityFacade.TYPE, RootEntityFacade.class); addEntityFacade(StreamFacade.TYPE_V1, StreamFacade.class); + addEntityFacade(StreamReferenceFacade.TYPE_V1, StreamReferenceFacade.class); addEntityFacade(DashboardFacade.TYPE_V2, DashboardFacade.class); addEntityFacade(DashboardV1Facade.TYPE_V1, DashboardV1Facade.class); addEntityFacade(SearchFacade.TYPE_V1, SearchFacade.class); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/PipelineFacade.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/PipelineFacade.java index 648a8eb7b351..af23ae3356a5 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/PipelineFacade.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/PipelineFacade.java @@ -64,6 +64,9 @@ import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityIdOrThrow; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntity; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; public class PipelineFacade implements EntityFacade { private static final Logger LOG = LoggerFactory.getLogger(PipelineFacade.class); @@ -112,7 +115,7 @@ Entity exportNativeEntity(PipelineDao pipelineDao, EntityDescriptorIds entityDes private Set connectedStreams(String pipelineId, EntityDescriptorIds entityDescriptorIds) { final Set connections = connectionsService.loadByPipelineId(pipelineId); return connections.stream() - .map(pipelineConnections -> entityDescriptorIds.getOrThrow(pipelineConnections.streamId(), ModelTypes.STREAM_V1)) + .map(pipelineConnections -> getStreamEntityIdOrThrow(pipelineConnections.streamId(), entityDescriptorIds)) .map(ValueReference::of) .collect(Collectors.toSet()); } @@ -157,7 +160,7 @@ private NativeEntity decode(EntityV1 entity, private Set connectedStreams(Set connectedStreamEntities, Map nativeEntities) { final ImmutableSet.Builder streams = ImmutableSet.builder(); for (EntityDescriptor descriptor : connectedStreamEntities) { - final Object stream = nativeEntities.get(descriptor); + final Object stream = resolveStreamEntityObject(descriptor.id().id(), nativeEntities); if (stream instanceof Stream) { streams.add((Stream) stream); } else { @@ -283,7 +286,7 @@ public Graph resolveNativeEntity(EntityDescriptor entityDescri pipelineConnections.stream() .map(PipelineConnections::streamId) .map(ModelId::of) - .map(id -> EntityDescriptor.create(id, ModelTypes.STREAM_V1)) + .map(id -> EntityDescriptor.create(id, ModelTypes.STREAM_REF_V1)) .forEach(stream -> mutableGraph.putEdge(entityDescriptor, stream)); } catch (NotFoundException e) { LOG.debug("Couldn't find pipeline {}", entityDescriptor, e); @@ -321,9 +324,7 @@ private Graph resolveForInstallation(EntityV1 entity, pipelineEntity.connectedStreams().stream() .map(valueReference -> valueReference.asString(parameters)) - .map(ModelId::of) - .map(modelId -> EntityDescriptor.create(modelId, ModelTypes.STREAM_V1)) - .map(entities::get) + .map(id -> resolveStreamEntity(id, entities)) .filter(Objects::nonNull) .forEach(streamEntity -> mutableGraph.putEdge(entity, streamEntity)); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/StreamReferenceFacade.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/StreamReferenceFacade.java new file mode 100644 index 000000000000..fbd22748528b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/StreamReferenceFacade.java @@ -0,0 +1,197 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.contentpacks.facades; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.graph.Graph; +import com.google.common.graph.GraphBuilder; +import com.google.common.graph.ImmutableGraph; +import com.google.common.graph.MutableGraph; +import org.graylog.events.legacy.V20190722150700_LegacyAlertConditionMigration; +import org.graylog2.contentpacks.EntityDescriptorIds; +import org.graylog2.contentpacks.exceptions.ContentPackException; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelType; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityDescriptor; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.contentpacks.model.entities.EntityV1; +import org.graylog2.contentpacks.model.entities.NativeEntity; +import org.graylog2.contentpacks.model.entities.StreamReferenceEntity; +import org.graylog2.contentpacks.model.entities.references.ValueReference; +import org.graylog2.database.NotFoundException; +import org.graylog2.indexer.indexset.IndexSetService; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.shared.users.UserService; +import org.graylog2.streams.StreamRuleService; +import org.graylog2.streams.StreamService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class StreamReferenceFacade extends StreamFacade { + private static final Logger LOG = LoggerFactory.getLogger(StreamReferenceFacade.class); + + public static final ModelType TYPE_V1 = ModelTypes.STREAM_REF_V1; + + private final ObjectMapper objectMapper; + private final StreamService streamService; + + @Inject + public StreamReferenceFacade(ObjectMapper objectMapper, StreamService streamService, StreamRuleService streamRuleService, V20190722150700_LegacyAlertConditionMigration legacyAlertsMigration, IndexSetService indexSetService, UserService userService) { + super(objectMapper, streamService, streamRuleService, legacyAlertsMigration, indexSetService, userService); + this.objectMapper = objectMapper; + this.streamService = streamService; + } + + @Override + public Graph resolveNativeEntity(EntityDescriptor entityDescriptor) { + final MutableGraph mutableGraph = GraphBuilder.directed().build(); + mutableGraph.addNode(EntityDescriptor.create(entityDescriptor.id().id(), ModelTypes.STREAM_REF_V1)); + return ImmutableGraph.copyOf(mutableGraph); + } + + @Override + public Optional exportEntity(EntityDescriptor entityDescriptor, EntityDescriptorIds entityDescriptorIds) { + final ModelId modelId = entityDescriptor.id(); + + // If we are already exporting the actual Stream, we don't need to export the title entity too. + if (entityDescriptorIds.get(modelId.id(), ModelTypes.STREAM_V1).isPresent()) { + return Optional.empty(); + } + + try { + final Stream stream = streamService.load(modelId.id()); + return Optional.of(exportNativeEntity(stream, entityDescriptorIds)); + } catch (NotFoundException e) { + LOG.debug("Couldn't find stream {}", entityDescriptor, e); + return Optional.empty(); + } + } + + @VisibleForTesting + Entity exportNativeEntity(Stream stream, EntityDescriptorIds entityDescriptorIds) { + final StreamReferenceEntity streamEntity = StreamReferenceEntity.create(ValueReference.of(stream.getTitle())); + + final JsonNode data = objectMapper.convertValue(streamEntity, JsonNode.class); + return EntityV1.builder() + .id(ModelId.of(entityDescriptorIds.getOrThrow(stream.getId(), ModelTypes.STREAM_REF_V1))) + .type(ModelTypes.STREAM_REF_V1) + .data(data) + .build(); + } + + @Override + public Graph resolveForInstallation(Entity entity, + Map parameters, + Map entities) { + if (entity instanceof EntityV1) { + final MutableGraph mutableGraph = GraphBuilder.directed().build(); + mutableGraph.addNode(entity); + return ImmutableGraph.copyOf(mutableGraph); + } else { + throw new IllegalArgumentException("Unsupported entity version: " + entity.getClass()); + } + } + + @Override + public Optional> findExisting(Entity entity, Map parameters) { + if (entity instanceof EntityV1) { + return findExisting((EntityV1) entity, parameters); + } else { + throw new IllegalArgumentException("Unsupported entity version: " + entity.getClass()); + } + } + + @Override + public NativeEntity createNativeEntity(Entity entity, + Map parameters, + Map nativeEntities, + String username) { + if (entity instanceof EntityV1) { + // Throw an exception if this is reached. Install process should fail earlier if no existing Stream found. + // A native entity cannot be created as this is only a reference to an existing stream. + final StreamReferenceEntity streamEntity = objectMapper.convertValue(((EntityV1) entity).data(), StreamReferenceEntity.class); + throw new ContentPackException("Stream with title <" + streamEntity.title().asString(parameters) + "> does not exist!"); + } else { + throw new IllegalArgumentException("Unsupported entity version: " + entity.getClass()); + } + } + + private Optional> findExisting(EntityV1 entity, Map parameters) { + final StreamReferenceEntity streamEntity = objectMapper.convertValue(entity.data(), StreamReferenceEntity.class); + final List streams = streamService.loadAllByTitle(streamEntity.title().asString()); + if (streams.size() == 1) { + final Stream stream = streams.get(0); + return Optional.of(NativeEntity.create(entity.id(), stream.getId(), ModelTypes.STREAM_V1, stream.getTitle(), stream)); + } else { + throw new ContentPackException(streams.isEmpty() + ? "Stream with title <" + streamEntity.title().asString(parameters) + "> does not exist!" + : "Multiple Streams with title <" + streamEntity.title().asString(parameters) + "> exist!"); + } + } + + @Override + public EntityExcerpt createExcerpt(Stream stream) { + return EntityExcerpt.builder() + .id(ModelId.of(stream.getTitle())) + .type(ModelTypes.STREAM_REF_V1) + .title(stream.getTitle()) + .build(); + } + + @Override + public Set listEntityExcerpts() { + return streamService.loadAll().stream() + .map(this::createExcerpt) + .collect(Collectors.toSet()); + } + + public static Entity resolveStreamEntity(String id, Map entities) { + return (Entity) resolveStreamEntityObject(id, entities); + } + + public static Object resolveStreamEntityObject(String id, Map entities) { + Object streamEntity = entities.get(EntityDescriptor.create(id, ModelTypes.STREAM_V1)); + if (streamEntity == null) { + streamEntity = entities.get(EntityDescriptor.create(id, ModelTypes.STREAM_REF_V1)); + } + return streamEntity; + } + + public static Optional getStreamEntityId(String id, EntityDescriptorIds entityDescriptorIds) { + Optional descriptor = entityDescriptorIds.get(id, ModelTypes.STREAM_V1); + if (descriptor.isEmpty()) { + descriptor = entityDescriptorIds.get(id, ModelTypes.STREAM_REF_V1); + } + return descriptor; + } + + public static String getStreamEntityIdOrThrow(String id, EntityDescriptorIds entityDescriptorIds) { + return getStreamEntityId(id, entityDescriptorIds).orElseThrow(() -> + new ContentPackException("Couldn't find entity " + id + "/" + ModelTypes.STREAM_V1 + " or " + ModelTypes.STREAM_REF_V1)); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/ViewFacade.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/ViewFacade.java index 8b988bf03963..7b93ce8a8a22 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/ViewFacade.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/ViewFacade.java @@ -58,6 +58,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntity; + public abstract class ViewFacade implements EntityWithExcerptFacade { private static final Logger LOG = LoggerFactory.getLogger(ViewFacade.class); @@ -188,7 +190,7 @@ public Graph resolveNativeEntity(EntityDescriptor entityDescri orElseThrow(() -> new NoSuchElementException("Could not find view with id " + modelId.id())); final Search search = searchDbService.get(viewSummaryDTO.searchId()). orElseThrow(() -> new NoSuchElementException("Could not find search with id " + viewSummaryDTO.searchId())); - search.usedStreamIds().stream().map(s -> EntityDescriptor.create(s, ModelTypes.STREAM_V1)) + search.usedStreamIds().stream().map(s -> EntityDescriptor.create(s, ModelTypes.STREAM_REF_V1)) .forEach(streamDescriptor -> mutableGraph.putEdge(entityDescriptor, streamDescriptor)); return ImmutableGraph.copyOf(mutableGraph); } @@ -216,8 +218,13 @@ protected Graph resolveViewEntity(EntityV1 entity, final MutableGraph mutableGraph = GraphBuilder.directed().build(); mutableGraph.addNode(entity); viewEntity.search().usedStreamIds().stream() - .map(s -> EntityDescriptor.create(s, ModelTypes.STREAM_V1)) - .map(entities::get) + .map(id -> resolveStreamEntity(id, entities)) + .filter(Objects::nonNull) + .forEach(stream -> mutableGraph.putEdge(entity, stream)); + viewEntity.state().values().stream() + .flatMap(s -> s.widgets().stream()) + .flatMap(w -> w.streams().stream()) + .map(id -> resolveStreamEntity(id, entities)) .filter(Objects::nonNull) .forEach(stream -> mutableGraph.putEdge(entity, stream)); return ImmutableGraph.copyOf(mutableGraph); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java index b1330c5676e0..1dd9fdbb6da8 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java @@ -29,6 +29,7 @@ public interface ModelTypes { ModelType PIPELINE_RULE_V1 = ModelType.of("pipeline_rule", "1"); ModelType ROOT = ModelType.of("virtual-root", "1"); ModelType STREAM_V1 = ModelType.of("stream", "1"); + ModelType STREAM_REF_V1 = ModelType.of("stream_title", "1"); ModelType EVENT_DEFINITION_V1 = ModelType.of("event_definition", "1"); ModelType NOTIFICATION_V1 = ModelType.of("notification", "1"); ModelType DASHBOARD_V1 = ModelType.of("dashboard", "1"); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java index 639026168df4..98f5765e9425 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java @@ -34,7 +34,6 @@ import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter; import org.graylog2.contentpacks.NativeEntityConverter; import org.graylog2.contentpacks.exceptions.ContentPackException; -import org.graylog2.contentpacks.model.ModelTypes; import org.graylog2.contentpacks.model.entities.references.ValueReference; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.streams.Stream; @@ -53,6 +52,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableSortedSet.of; import static java.util.stream.Collectors.toSet; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; @AutoValue @JsonAutoDetect @@ -149,8 +149,7 @@ private Filter shallowMappedFilter(Map nativeEntities) .map(filter -> { if (filter.type().matches(StreamFilter.NAME)) { final StreamFilter streamFilter = (StreamFilter) filter; - final Stream stream = (Stream) nativeEntities.get( - EntityDescriptor.create(streamFilter.streamId(), ModelTypes.STREAM_V1)); + final Stream stream = (Stream) resolveStreamEntityObject(streamFilter.streamId(), nativeEntities); if (Objects.isNull(stream)) { throw new ContentPackException("Could not find matching stream id: " + streamFilter.streamId()); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java index 674bbfd4742a..60584551c245 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java @@ -27,7 +27,6 @@ import org.graylog.plugins.views.search.timeranges.DerivedTimeRange; import org.graylog2.contentpacks.NativeEntityConverter; import org.graylog2.contentpacks.exceptions.ContentPackException; -import org.graylog2.contentpacks.model.ModelTypes; import org.graylog2.contentpacks.model.entities.references.ValueReference; import org.graylog2.plugin.streams.Stream; @@ -40,6 +39,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; + @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, @@ -193,8 +194,7 @@ public SearchType toNativeEntity(Map parameters, Map mappedStreams(Map nativeEntities) { return streams().stream() - .map(s -> EntityDescriptor.create(s, ModelTypes.STREAM_V1)) - .map(nativeEntities::get) + .map(id -> resolveStreamEntityObject(id, nativeEntities)) .map(object -> { if (object == null) { throw new ContentPackException("Missing Stream for event definition"); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/StreamReferenceEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/StreamReferenceEntity.java new file mode 100644 index 000000000000..9dcd121806be --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/StreamReferenceEntity.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.contentpacks.model.entities; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.graylog.autovalue.WithBeanGetter; +import org.graylog2.contentpacks.model.entities.references.ValueReference; + +import javax.validation.constraints.NotBlank; + +@AutoValue +@WithBeanGetter +@JsonAutoDetect +public abstract class StreamReferenceEntity { + @JsonProperty("title") + @NotBlank + public abstract ValueReference title(); + + @JsonCreator + public static StreamReferenceEntity create( + @JsonProperty("title") @NotBlank ValueReference title) { + return new AutoValue_StreamReferenceEntity(title); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java index ad0cfb83fdc3..44835d4ab26c 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java @@ -48,7 +48,6 @@ import org.graylog.plugins.views.search.views.widgets.aggregation.sort.SortConfigDTO; import org.graylog2.contentpacks.NativeEntityConverter; import org.graylog2.contentpacks.exceptions.ContentPackException; -import org.graylog2.contentpacks.model.ModelTypes; import org.graylog2.contentpacks.model.entities.references.ValueReference; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.streams.Stream; @@ -64,6 +63,8 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; + @AutoValue @JsonDeserialize(builder = WidgetEntity.Builder.class) @WithBeanGetter @@ -155,8 +156,7 @@ public WidgetDTO toNativeEntity(Map parameters, Map EntityDescriptor.create(id, ModelTypes.STREAM_V1)) - .map(nativeEntities::get) + .map(id -> resolveStreamEntityObject(id, nativeEntities)) .map(object -> { if (object == null) { throw new ContentPackException("Missing Stream for widget entity"); diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java index d25e79639627..2bd41ec6753c 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java @@ -52,6 +52,10 @@ public interface StreamService extends PersistedService { List loadAllEnabled(); + default List loadAllByTitle(String title) { + return loadAll().stream().filter(s -> title.equals(s.getTitle())).toList(); + } + /** * @return the total number of streams */ diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java index 3621b4ff5966..815ce8390b76 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java @@ -175,6 +175,11 @@ public List loadAllEnabled(Map additionalQueryOpts) { return loadAll(additionalQueryOpts); } + @Override + public List loadAllByTitle(String title) { + return loadAll(QueryBuilder.start(StreamImpl.FIELD_TITLE).is(title).get()); + } + @Override public List loadAll() { return loadAll(Collections.emptyMap()); diff --git a/graylog2-server/src/test/java/org/graylog/events/contentpack/facade/EventDefinitionFacadeTest.java b/graylog2-server/src/test/java/org/graylog/events/contentpack/facade/EventDefinitionFacadeTest.java index 283730487cb7..8c89e5638f54 100644 --- a/graylog2-server/src/test/java/org/graylog/events/contentpack/facade/EventDefinitionFacadeTest.java +++ b/graylog2-server/src/test/java/org/graylog/events/contentpack/facade/EventDefinitionFacadeTest.java @@ -355,7 +355,7 @@ public void resolveNativeEntity() { EntityDescriptor eventDescriptor = EntityDescriptor .create("5d4032513d2746703d1467f6", ModelTypes.EVENT_DEFINITION_V1); EntityDescriptor streamDescriptor = EntityDescriptor - .create("5cdab2293d27467fbe9e8a72", ModelTypes.STREAM_V1); + .create("5cdab2293d27467fbe9e8a72", ModelTypes.STREAM_REF_V1); Set expectedNodes = ImmutableSet.of(eventDescriptor, streamDescriptor); Graph graph = facade.resolveNativeEntity(eventDescriptor); assertThat(graph).isNotNull(); diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/PipelineFacadeTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/PipelineFacadeTest.java index dd2b44d3c290..af25bde56b2b 100644 --- a/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/PipelineFacadeTest.java +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/PipelineFacadeTest.java @@ -289,7 +289,7 @@ public void resolveEntityDescriptor() { final Graph graph = facade.resolveNativeEntity(descriptor); assertThat(graph.nodes()).containsOnly( descriptor, - EntityDescriptor.create("5adf23894b900a0fdb4e517d", ModelTypes.STREAM_V1), + EntityDescriptor.create("5adf23894b900a0fdb4e517d", ModelTypes.STREAM_REF_V1), EntityDescriptor.create("2342353045938450345", ModelTypes.PIPELINE_RULE_V1)); } @@ -387,7 +387,7 @@ public void resolve() { final Graph graph = facade.resolveNativeEntity(pipelineEntity); - final EntityDescriptor streamEntity = EntityDescriptor.create("5adf23894b900a0fdb4e517d", ModelTypes.STREAM_V1); + final EntityDescriptor streamEntity = EntityDescriptor.create("5adf23894b900a0fdb4e517d", ModelTypes.STREAM_REF_V1); final EntityDescriptor ruleEntity1 = EntityDescriptor.create("2342353045938450345", ModelTypes.PIPELINE_RULE_V1); final EntityDescriptor ruleEntity2 = EntityDescriptor.create("2342353045938450346", ModelTypes.PIPELINE_RULE_V1); assertThat(graph.nodes()) diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/ViewFacadeTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/ViewFacadeTest.java index 83d367e948c8..bf239e6bca05 100644 --- a/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/ViewFacadeTest.java +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/facades/ViewFacadeTest.java @@ -278,7 +278,7 @@ public void itShouldResolveDependencyForInstallation() throws Exception { @Test @MongoDBFixtures("ViewFacadeTest.json") public void itShouldResolveDependencyForCreation() { - final EntityDescriptor streamEntityDescriptor = EntityDescriptor.create(streamId, ModelTypes.STREAM_V1); + final EntityDescriptor streamEntityDescriptor = EntityDescriptor.create(streamId, ModelTypes.STREAM_REF_V1); final EntityDescriptor viewEntityDescriptor = EntityDescriptor.create(viewId, ModelTypes.SEARCH_V1); Graph graph = facade.resolveNativeEntity(viewEntityDescriptor);