Use Stream Names for Content Packs (#17089) (#17609)

* Use stream names for content packs * Cleanup * Fix install * cleanup * remove unused import * Handle multiple streams better, minor name refactoring * Add handling for the rest of the entity types that reference streams * Fix line spacing * Add missing license header * Fix failing tests * Add changelog entry * Return StreamReference Entity Exerpts to front end to ensure proper handling of existing entities * Use Stream title to generate excerpt IDs to avoid collision with regular Streams. Irrellevant to content packs but was causing problems for start page and related tests. * Add default naive implementation of StreamService loadAllByTitle() * Resolve stream deps for dashboard widgets
Graylog2 · Dec 8, 2023 · 64c8be5 · 64c8be5
1 parent 2c0db86
commit 64c8be5
Show file tree

Hide file tree

Showing 20 changed files with 315 additions and 41 deletions.
diff --git a/changelog/unreleased/issue-16885.toml b/changelog/unreleased/issue-16885.toml
@@ -0,0 +1,19 @@
+type = "changed"
+message = "Changed Content Packs handling to allow import/export of entites that reference Streams by title."
+
+pulls = ["16743"]
+
+details.user = """
+Previously it was not possible to create a Content Pack with Stream scoped entities without also exporting the referenced Stream,
+and potentially duplicating the Stream.
+
+For example, if a user had a Dashboard with a widget that was scoped to "stream_xyz" that they wished to create a
+Content Pack with to use on another system, there were two options:
+- Remove the Stream from the Dashboard widget before export, then re-associate the Stream with the Dashboard after uploading.
+- Export the Stream along with the Dashboard, in which case a new "stream_xyz" would be created on the uploading system
+    (whether it already existed or not).
+
+This change allows users to create a Content Pack with a "stream_xyz" scoped Dashboard, referencing the Stream by title only.
+When uploaded and installed, the Content Pack will resolve the existing stream with title "stream_xyz",
+and associate it to the new Dashboard.
+"""
diff --git a/...n/java/org/graylog/events/contentpack/entities/AggregationEventProcessorConfigEntity.java b/...n/java/org/graylog/events/contentpack/entities/AggregationEventProcessorConfigEntity.java
@@ -27,8 +27,6 @@
 import org.graylog.events.processor.aggregation.AggregationConditions;
 import org.graylog.events.processor.aggregation.AggregationEventProcessorConfig;
 import org.graylog2.contentpacks.exceptions.ContentPackException;
-import org.graylog2.contentpacks.model.ModelId;
-import org.graylog2.contentpacks.model.ModelTypes;
 import org.graylog2.contentpacks.model.entities.Entity;
 import org.graylog2.contentpacks.model.entities.EntityDescriptor;
 import org.graylog2.contentpacks.model.entities.EntityV1;
@@ -42,6 +40,9 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntity;
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject;
+
 @AutoValue
 @JsonTypeName(AggregationEventProcessorConfigEntity.TYPE_NAME)
 @JsonDeserialize(builder = AggregationEventProcessorConfigEntity.Builder.class)
@@ -129,8 +130,7 @@ public EventProcessorConfig toNativeEntity(Map<String, ValueReference> parameter
                                                Map<EntityDescriptor, Object> nativeEntities) {
         final ImmutableSet<String> streamSet = ImmutableSet.copyOf(
                 streams().stream()
-                        .map(id -> EntityDescriptor.create(id, ModelTypes.STREAM_V1))
-                        .map(nativeEntities::get)
+                        .map(id -> resolveStreamEntityObject(id, nativeEntities))
                         .map(object -> {
                             if (object == null) {
                                 throw new ContentPackException("Missing Stream for event definition");
@@ -162,9 +162,7 @@ public void resolveForInstallation(EntityV1 entity,
                                        Map<EntityDescriptor, Entity> entities,
                                        MutableGraph<Entity> graph) {
         streams().stream()
-                .map(ModelId::of)
-                .map(modelId -> EntityDescriptor.create(modelId, ModelTypes.STREAM_V1))
-                .map(entities::get)
+                .map(id -> resolveStreamEntity(id, entities))
                 .filter(Objects::nonNull)
                 .forEach(stream -> graph.putEdge(entity, stream));
     }

diff --git a/...c/main/java/org/graylog/events/processor/aggregation/AggregationEventProcessorConfig.java b/...c/main/java/org/graylog/events/processor/aggregation/AggregationEventProcessorConfig.java
@@ -53,6 +53,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityId;
 import static org.graylog2.shared.utilities.StringUtils.f;
 
 @AutoValue
@@ -247,7 +248,7 @@ private void checkEventLimitGreaterZero(ValidationResult validationResult) {
     @Override
     public EventProcessorConfigEntity toContentPackEntity(EntityDescriptorIds entityDescriptorIds) {
         final ImmutableSet<String> streamRefs = ImmutableSet.copyOf(streams().stream()
-                .map(streamId -> entityDescriptorIds.get(streamId, ModelTypes.STREAM_V1))
+                .map(streamId -> getStreamEntityId(streamId, entityDescriptorIds))
                 .filter(Optional::isPresent)
                 .map(Optional::get)
                 .collect(Collectors.toSet()));
@@ -269,7 +270,7 @@ public void resolveNativeEntity(EntityDescriptor entityDescriptor, MutableGraph<
         streams().forEach(streamId -> {
             final EntityDescriptor depStream = EntityDescriptor.builder()
                     .id(ModelId.of(streamId))
-                    .type(ModelTypes.STREAM_V1)
+                    .type(ModelTypes.STREAM_REF_V1)
                     .build();
             mutableGraph.putEdge(entityDescriptor, depStream);
         });

diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java
@@ -40,7 +40,6 @@
 import org.graylog.plugins.views.search.searchfilters.model.UsesSearchFilters;
 import org.graylog2.contentpacks.ContentPackable;
 import org.graylog2.contentpacks.EntityDescriptorIds;
-import org.graylog2.contentpacks.model.ModelTypes;
 import org.graylog2.contentpacks.model.entities.QueryEntity;
 import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
@@ -60,6 +59,7 @@
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.ImmutableSortedSet.of;
 import static java.util.stream.Collectors.toSet;
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityIdOrThrow;
 
 @AutoValue
 @JsonAutoDetect
@@ -290,8 +290,7 @@ private Filter shallowMappedFilter(EntityDescriptorIds entityDescriptorIds) {
                             .map(filter -> {
                                 if (filter.type().equals(StreamFilter.NAME)) {
                                     final StreamFilter streamFilter = (StreamFilter) filter;
-                                    final String streamId = entityDescriptorIds.
-                                            getOrThrow(streamFilter.streamId(), ModelTypes.STREAM_V1);
+                                    final String streamId = getStreamEntityIdOrThrow(streamFilter.streamId(), entityDescriptorIds);
                                     return streamFilter.toBuilder().streamId(streamId).build();
                                 }
                                 return filter;

diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchType.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchType.java
@@ -29,8 +29,6 @@
 import org.graylog2.contentpacks.ContentPackable;
 import org.graylog2.contentpacks.EntityDescriptorIds;
 import org.graylog2.contentpacks.exceptions.ContentPackException;
-import org.graylog2.contentpacks.model.ModelTypes;
-import org.graylog2.contentpacks.model.entities.EntityDescriptor;
 import org.graylog2.contentpacks.model.entities.SearchTypeEntity;
 
 import javax.annotation.Nullable;
@@ -42,6 +40,8 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityId;
+
 /**
  * A search type represents parts of a query that generates a {@see Result result}.
  * <p>
@@ -243,7 +243,7 @@ public SearchTypeEntity toContentPackEntity(EntityDescriptorIds entityDescriptor
 
     default Set<String> mappedStreams(EntityDescriptorIds entityDescriptorIds) {
         return streams().stream()
-                .map(streamId -> entityDescriptorIds.get(EntityDescriptor.create(streamId, ModelTypes.STREAM_V1)))
+                .map(streamId -> getStreamEntityId(streamId, entityDescriptorIds))
                 .map(optionalStreamId -> optionalStreamId.orElseThrow(() ->
                         new ContentPackException("Did not find matching stream id")))
                 .collect(Collectors.toSet());

diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/WidgetDTO.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/WidgetDTO.java
@@ -27,8 +27,6 @@
 import org.graylog.plugins.views.search.searchfilters.model.UsesSearchFilters;
 import org.graylog2.contentpacks.ContentPackable;
 import org.graylog2.contentpacks.EntityDescriptorIds;
-import org.graylog2.contentpacks.model.ModelTypes;
-import org.graylog2.contentpacks.model.entities.EntityDescriptor;
 import org.graylog2.contentpacks.model.entities.WidgetEntity;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
 
@@ -39,6 +37,8 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityId;
+
 @AutoValue
 @JsonDeserialize(builder = WidgetDTO.Builder.class)
 @WithBeanGetter
@@ -128,8 +128,8 @@ static Builder builder() {
 
     @Override
     public WidgetEntity toContentPackEntity(EntityDescriptorIds entityDescriptorIds) {
-        Set<String> mappedStreams = streams().stream().map(streamId ->
-                entityDescriptorIds.get(EntityDescriptor.create(streamId, ModelTypes.STREAM_V1)))
+        Set<String> mappedStreams = streams().stream()
+                .map(streamId -> getStreamEntityId(streamId, entityDescriptorIds))
                 .filter(Optional::isPresent)
                 .map(Optional::get)
                 .collect(Collectors.toSet());

diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java
@@ -32,6 +32,7 @@
 import org.graylog2.contentpacks.facades.SidecarCollectorConfigurationFacade;
 import org.graylog2.contentpacks.facades.SidecarCollectorFacade;
 import org.graylog2.contentpacks.facades.StreamFacade;
+import org.graylog2.contentpacks.facades.StreamReferenceFacade;
 import org.graylog2.contentpacks.facades.UrlWhitelistFacade;
 import org.graylog2.contentpacks.facades.dashboardV1.DashboardV1Facade;
 import org.graylog2.contentpacks.jersey.ModelIdParamConverter;
@@ -61,6 +62,7 @@ protected void configure() {
         addEntityFacade(PipelineRuleFacade.TYPE_V1, PipelineRuleFacade.class);
         addEntityFacade(RootEntityFacade.TYPE, RootEntityFacade.class);
         addEntityFacade(StreamFacade.TYPE_V1, StreamFacade.class);
+        addEntityFacade(StreamReferenceFacade.TYPE_V1, StreamReferenceFacade.class);
         addEntityFacade(DashboardFacade.TYPE_V2, DashboardFacade.class);
         addEntityFacade(DashboardV1Facade.TYPE_V1, DashboardV1Facade.class);
         addEntityFacade(SearchFacade.TYPE_V1, SearchFacade.class);

diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/PipelineFacade.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/facades/PipelineFacade.java
@@ -64,6 +64,9 @@
 import java.util.stream.Collectors;
 
 import static java.util.Objects.requireNonNull;
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.getStreamEntityIdOrThrow;
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntity;
+import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject;
 
 public class PipelineFacade implements EntityFacade<PipelineDao> {
     private static final Logger LOG = LoggerFactory.getLogger(PipelineFacade.class);
@@ -112,7 +115,7 @@ Entity exportNativeEntity(PipelineDao pipelineDao, EntityDescriptorIds entityDes
     private Set<ValueReference> connectedStreams(String pipelineId, EntityDescriptorIds entityDescriptorIds) {
         final Set<PipelineConnections> connections = connectionsService.loadByPipelineId(pipelineId);
         return connections.stream()
-                .map(pipelineConnections -> entityDescriptorIds.getOrThrow(pipelineConnections.streamId(), ModelTypes.STREAM_V1))
+                .map(pipelineConnections -> getStreamEntityIdOrThrow(pipelineConnections.streamId(), entityDescriptorIds))
                 .map(ValueReference::of)
                 .collect(Collectors.toSet());
     }
@@ -157,7 +160,7 @@ private NativeEntity<PipelineDao> decode(EntityV1 entity,
     private Set<Stream> connectedStreams(Set<EntityDescriptor> connectedStreamEntities, Map<EntityDescriptor, Object> nativeEntities) {
         final ImmutableSet.Builder<Stream> streams = ImmutableSet.builder();
         for (EntityDescriptor descriptor : connectedStreamEntities) {
-            final Object stream = nativeEntities.get(descriptor);
+            final Object stream = resolveStreamEntityObject(descriptor.id().id(), nativeEntities);
             if (stream instanceof Stream) {
                 streams.add((Stream) stream);
             } else {
@@ -283,7 +286,7 @@ public Graph<EntityDescriptor> resolveNativeEntity(EntityDescriptor entityDescri
             pipelineConnections.stream()
                     .map(PipelineConnections::streamId)
                     .map(ModelId::of)
-                    .map(id -> EntityDescriptor.create(id, ModelTypes.STREAM_V1))
+                    .map(id -> EntityDescriptor.create(id, ModelTypes.STREAM_REF_V1))
                     .forEach(stream -> mutableGraph.putEdge(entityDescriptor, stream));
         } catch (NotFoundException e) {
             LOG.debug("Couldn't find pipeline {}", entityDescriptor, e);
@@ -321,9 +324,7 @@ private Graph<Entity> resolveForInstallation(EntityV1 entity,
 
         pipelineEntity.connectedStreams().stream()
                 .map(valueReference -> valueReference.asString(parameters))
-                .map(ModelId::of)
-                .map(modelId -> EntityDescriptor.create(modelId, ModelTypes.STREAM_V1))
-                .map(entities::get)
+                .map(id -> resolveStreamEntity(id, entities))
                 .filter(Objects::nonNull)
                 .forEach(streamEntity -> mutableGraph.putEdge(entity, streamEntity));