Merge branch 'master' into clipboard-icon-button

changelog/unreleased/issue-20955.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fix unescaped double quotes in map and collection typed fields in Custom HTTP Notification JSON body."
+
+issues = ["20955"]
+pulls = ["21167"]

data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfiguration.java

@@ -16,7 +16,6 @@
  */
 package org.graylog.datanode.configuration;
 
-import org.graylog.datanode.OpensearchDistribution;
 import org.graylog2.security.IndexerJwtAuthTokenProvider;
 
 /**
@@ -27,7 +26,6 @@ public record DatanodeConfiguration(
         OpensearchDistributionProvider opensearchDistributionProvider,
         DatanodeDirectories datanodeDirectories,
         int processLogsBufferSize,
-        String opensearchHeap,
         IndexerJwtAuthTokenProvider indexerJwtAuthTokenProvider
 ) {
 }

data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfigurationProvider.java

@@ -40,7 +40,6 @@ public DatanodeConfigurationProvider(
                 opensearchDistributionProvider,
                 DatanodeDirectories.fromConfiguration(localConfiguration, nodeId),
                 localConfiguration.getProcessLogsBufferSize(),
-                localConfiguration.getOpensearchHeap(),
                 jwtTokenProvider
         );
     }

data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java

@@ -43,8 +43,8 @@ public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationP
         return DatanodeConfigurationPart.builder()
                 .properties(commonOpensearchConfig(buildParams))
                 .nodeRoles(localConfiguration.getNodeRoles())
-                .javaOpt("-Xms%s".formatted(datanodeConfiguration.opensearchHeap()))
-                .javaOpt("-Xmx%s".formatted(datanodeConfiguration.opensearchHeap()))
+                .javaOpt("-Xms%s".formatted(localConfiguration.getOpensearchHeap()))
+                .javaOpt("-Xmx%s".formatted(localConfiguration.getOpensearchHeap()))
                 .javaOpt("-Dopensearch.transport.cname_in_publish_address=true")
                 .build();
     }

full-backend-tests/src/test/java/org/graylog2/inputs/InputCreationIT.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.inputs;
+
+import org.graylog.testing.completebackend.Lifecycle;
+import org.graylog.testing.completebackend.apis.GraylogApis;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration;
+
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+@ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS)
+public class InputCreationIT {
+
+    private final GraylogApis apis;
+
+    public InputCreationIT(GraylogApis apis) {
+        this.apis = apis;
+    }
+
+    @ContainerMatrixTest
+    void testHttpRandomInputCreation() {
+        String inputId = apis.inputs().createGlobalInput("testInput",
+                "org.graylog2.inputs.random.FakeHttpMessageInput",
+                Map.of("sleep", 30,
+                        "sleep_deviation", 30,
+                        "source", "example.org"));
+        apis.inputs().getInput(inputId)
+                .assertThat().body("title", equalTo("testInput"));
+        apis.waitFor(() ->
+                        apis.inputs().getInputState(inputId)
+                                .extract().body().jsonPath().get("state")
+                                .equals("RUNNING"),
+                "Timed out waiting for HTTP Random Message Input to become available");
+        apis.inputs().deleteInput(inputId);
+    }
+
+    /**
+     * Test to make sure configuration encryption serialization/deserialization works
+     */
+    @ContainerMatrixTest
+    void testFailingAwsCloudTrailInputCreation() {
+        String inputId = apis.inputs().createGlobalInput("testInput",
+                "org.graylog.aws.inputs.cloudtrail.CloudTrailInput",
+                Map.of("aws_sqs_region", "us-east-1",
+                        "aws_s3_region", "us-east-1",
+                        "aws_sqs_queue_name", "invalid-queue-no-messages-read",
+                        "aws_access_key", "invalid-access-key",
+                        "aws_secret_key", "invalid-secret-key"));
+        apis.inputs().getInput(inputId)
+                .assertThat().body("attributes.aws_access_key", equalTo("invalid-access-key"));
+        apis.waitFor(() ->
+                        apis.inputs().getInputState(inputId)
+                                .extract().body().jsonPath().get("state")
+                                .equals("FAILING"),
+                "Timed out waiting for AWS CloudTrail Input to reach failing state");
+        apis.inputs().deleteInput(inputId);
+    }
+}

graylog-project-parent/pom.xml

@@ -614,7 +614,7 @@
                                 <plugin>
                                     <groupId>com.mebigfatguy.fb-contrib</groupId>
                                     <artifactId>fb-contrib</artifactId>
-                                    <version>7.6.8</version>
+                                    <version>7.6.9</version>
                                 </plugin>
                             </plugins>
                         </configuration>

graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java

@@ -277,7 +277,7 @@ public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext que
 
         //ES does not support per-request cancel_after_time_interval. We have to use simplified solution - the whole multi-search will be cancelled if it takes more than configured max. exec. time.
         final PlainActionFuture<MultiSearchResponse> mSearchFuture = client.cancellableMsearch(searches);
-        job.setSearchEngineTaskFuture(mSearchFuture);
+        job.setQueryExecutionFuture(query.id(), mSearchFuture);
         final List<MultiSearchResponse.Item> results = getResults(mSearchFuture, job.getCancelAfterSeconds(), searches.size());
 
         for (SearchType searchType : query.searchTypes()) {

graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java

@@ -283,7 +283,7 @@ public QueryResult doRun(SearchJob job, Query query, OSGeneratedQueryContext que
                 .toList();
 
         final PlainActionFuture<MultiSearchResponse> mSearchFuture = client.cancellableMsearch(searches);
-        job.setSearchEngineTaskFuture(mSearchFuture);
+        job.setQueryExecutionFuture(query.id(), mSearchFuture);
         final List<MultiSearchResponse.Item> results = getResults(mSearchFuture, searches.size());
 
         for (SearchType searchType : query.searchTypes()) {

graylog2-server/src/main/java/org/graylog/plugins/views/search/SearchJob.java

@@ -28,7 +28,9 @@
 import org.graylog.plugins.views.search.errors.SearchError;
 import org.graylog.plugins.views.search.rest.ExecutionInfo;
 
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -46,7 +48,7 @@ public class SearchJob implements ParameterProvider {
 
     private final Search search;
 
-    private Future<?> searchEngineTaskFuture;
+    private Map<String, Future<?>> queryExecutionFutures;
 
     private CompletableFuture<Void> resultFuture;
 
@@ -72,6 +74,7 @@ public SearchJob(String id,
         this.search = search;
         this.searchJobIdentifier = new SearchJobIdentifier(id, search.id(), owner, executingNodeId);
         this.cancelAfterSeconds = cancelAfterSeconds != null ? cancelAfterSeconds : NO_CANCELLATION;
+        this.queryExecutionFutures = new HashMap<>();
     }
 
     @JsonIgnore //covered by @JsonUnwrapped
@@ -120,14 +123,14 @@ public void addQueryResultFuture(String queryId, CompletableFuture<QueryResult>
     }
 
     @JsonIgnore
-    public void setSearchEngineTaskFuture(final Future<?> searchEngineTaskFuture) {
-        this.searchEngineTaskFuture = searchEngineTaskFuture;
+    public void setQueryExecutionFuture(final String queryId, final Future<?> future) {
+        this.queryExecutionFutures.put(queryId, future);
     }
 
     public void cancel() {
-        if (this.searchEngineTaskFuture != null) {
-            this.searchEngineTaskFuture.cancel(true);
-        }
+        this.queryExecutionFutures.values().stream()
+                .filter(Objects::nonNull)
+                .forEach(f -> f.cancel(true));
     }
 
     @JsonProperty("results")
@@ -141,8 +144,8 @@ public Map<String, QueryResult> results() {
 
     @JsonProperty("execution")
     public ExecutionInfo execution() {
-        final boolean isDone = (resultFuture == null || resultFuture.isDone()) && (searchEngineTaskFuture == null || searchEngineTaskFuture.isDone());
-        final boolean isCancelled = (searchEngineTaskFuture != null && searchEngineTaskFuture.isCancelled()) || (resultFuture != null && resultFuture.isCancelled());
+        final boolean isDone = (resultFuture == null || resultFuture.isDone()) && (queryExecutionFutures.values().stream().allMatch(f -> f == null || f.isDone()));
+        final boolean isCancelled = (queryExecutionFutures.values().stream().allMatch(f -> f != null && f.isCancelled()) || (resultFuture != null && resultFuture.isCancelled()));
         return new ExecutionInfo(isDone, isCancelled, !errors.isEmpty());
     }
 

graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java

@@ -50,6 +50,11 @@ default boolean withNodeIdFile() {
         return true;
     }
 
+    @Override
+    default boolean withInputs() {
+        return false;
+    }
+
     @Override
     default Set<ServerStatus.Capability> withCapabilities() {
         return Set.of(ServerStatus.Capability.SERVER);

graylog2-server/src/main/java/org/graylog2/Configuration.java

@@ -670,4 +670,9 @@ private static int defaultNumberOfOutputBufferProcessors() {
     public boolean withPlugins() {
         return true;
     }
+
+    @Override
+    public boolean withInputs() {
+        return true;
+    }
 }

graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java

@@ -51,6 +51,11 @@ public interface GraylogNodeConfiguration {
      */
     boolean withNodeIdFile();
 
+    /**
+     * Will only bind an InputConfigurationDeserializerModifier stub if there are no inputs configured
+     */
+    boolean withInputs();
+
     /**
      * Provides the {@link ServerStatus.Capability} to be used by ServerStatusBindings.
      */

graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java

@@ -78,7 +78,9 @@ protected void configure() {
         install(new ServerStatusBindings(configuration.withCapabilities()));
 
         bind(EncryptedValueService.class).asEagerSingleton();
-        bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig());
+        if (!configuration.withInputs()) {
+            bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig());
+        }
     }
 
     public Set<Object> getConfigurationBeans() {

graylog2-server/src/main/java/org/graylog2/bindings/providers/JsonSafeEngineProvider.java

@@ -23,6 +23,8 @@
 import jakarta.inject.Singleton;
 import org.apache.commons.lang.StringEscapeUtils;
 
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.Locale;
 import java.util.Map;
 
@@ -34,7 +36,11 @@ public class JsonSafeEngineProvider implements Provider<Engine> {
     public JsonSafeEngineProvider() {
         engine = Engine.createEngine();
         engine.registerRenderer(String.class, new JsonSafeRenderer());
+        engine.registerRenderer(Map.class, new JsonSafeMapRenderer());
+        engine.registerRenderer(Iterable.class, new JsonSafeIterableRenderer());
+        engine.registerRenderer(Collection.class, new JsonSafeCollectionRenderer());
     }
+
     @Override
     public Engine get() {
         return engine;
@@ -52,4 +58,55 @@ public String render(String s, Locale locale, Map<String, Object> map) {
             return StringEscapeUtils.escapeJava(s).replace("/", "\\/");
         }
     }
+
+    @SuppressWarnings("rawtypes")
+    private static class JsonSafeMapRenderer implements Renderer<Map> {
+
+        @Override
+        public String render(Map map, Locale locale, Map<String, Object> map2) {
+            final String renderedResult;
+
+            if (map.isEmpty()) {
+                renderedResult = "";
+            } else if (map.size() == 1) {
+                renderedResult = map.values().iterator().next().toString();
+            } else {
+                renderedResult = map.toString();
+            }
+            return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/");
+        }
+    }
+
+    private static class JsonSafeIterableRenderer implements Renderer<Iterable> {
+
+        @Override
+        public String render(Iterable iterable, Locale locale, Map<String, Object> model) {
+            final String renderedResult;
+
+            final Iterator<?> iterator = iterable.iterator();
+            renderedResult = iterator.hasNext() ? iterator.next().toString() : "";
+            return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/");
+
+        }
+
+    }
+
+    private static class JsonSafeCollectionRenderer implements Renderer<Collection> {
+
+        @Override
+        public String render(Collection collection, Locale locale, Map<String, Object> model) {
+            final String renderedResult;
+
+            if (collection.isEmpty()) {
+                renderedResult = "";
+            } else if (collection.size() == 1) {
+                renderedResult = collection.iterator().next().toString();
+            } else {
+                renderedResult = collection.toString();
+            }
+            return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/");
+
+        }
+
+    }
 }