Merge branch 'master' into datanode-list-page

.github/workflows/fix-linter-hints.yml

@@ -2,7 +2,7 @@ name: Fix linter hints
 
 on:
   schedule:
-    - cron: '0 */8 * * *'
+    - cron: '0 */8 * * MON-FRI'
 
   workflow_dispatch:
 
@@ -46,7 +46,8 @@ jobs:
         run: |
           /bin/sh ../.github/scripts/web-code-quality.sh | curl -X POST -H 'Content-Type: application/json' -d @- ${{ secrets.DEV_METRICS_URL }}
       - name: Create/Update Pull Request
-        uses: Graylog2/create-pull-request@7380612b49221684fefa025244f2ef4008ae50ad
+        id: pr-create
+        uses: Graylog2/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38
         with:
           title: Fixing linter hints automatically
           body: This PR was created by a job that is running periodically to find and fix linter hints.
@@ -55,3 +56,26 @@ jobs:
           committer: Dr. Lint-a-lot <[email protected]>
           commit-message: Running lint --fix
           delete-branch: true
+
+      - name: Get headRef and SHA of PR
+        id: pr-infos
+        if: ${{ steps.pr-create.outputs.pull-request-number && steps.pr-create.outputs.pull-request-operation != 'closed' }}
+        run: |
+          HEADREF=$(gh pr view ${{ steps.pr-create.outputs.pull-request-number }} --json headRefName --template '{{.headRefName}}')
+          echo "sha=$(git rev-parse origin/$HEADREF)" >> $GITHUB_OUTPUT
+          echo "headref=$HEADREF" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Request dispatched PR build
+        if: ${{ steps.pr-create.outputs.pull-request-number && steps.pr-create.outputs.pull-request-operation != 'closed' }}
+        run: >
+          gh workflow run -R Graylog2/graylog-project-internal pr-build.yml --ref master
+          -f caller_repo=${{ github.repository }}
+          -f caller_pr_nr=${{ steps.pr-create.outputs.pull-request-number }}
+          -f caller_base_branch=master
+          -f caller_head_branch=${{ steps.pr-infos.outputs.headref }}
+          -f head_sha=${{ steps.pr-infos.outputs.sha }}
+          -f initial_actor="Dr. Lint-a-lot"
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT_GRAYLOG_PROJECT_INTERNAL_WORKFLOW_RW }}

.github/workflows/update-browserslist-db.yml

@@ -2,7 +2,7 @@ name: Update browserslist DB.
 
 on: 
   schedule:
-    - cron: '0 0 1/14 * *'
+    - cron: '0 0 1/14 * MON-FRI'
 
   workflow_dispatch:
 
@@ -21,7 +21,8 @@ jobs:
         continue-on-error: true
         run: npx browserslist@latest --update-db
       - name: Create/Update Pull Request
-        uses: Graylog2/create-pull-request@7380612b49221684fefa025244f2ef4008ae50ad
+        id: pr-create
+        uses: Graylog2/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38
         with:
           title: Updating browserslist DB.
           body: This PR was created by a job that is running periodically to update the browserslist db automatically. See [here](https://github.com/browserslist/browserslist#browsers-data-updating) for details.
@@ -30,3 +31,26 @@ jobs:
           committer: Dr. Lint-a-lot <[email protected]>
           commit-message: Updating browserslist DB.
           delete-branch: true
+
+      - name: Get headRef and SHA of PR
+        id: pr-infos
+        if: ${{ steps.pr-create.outputs.pull-request-number && steps.pr-create.outputs.pull-request-operation != 'closed' }}
+        run: |
+          HEADREF=$(gh pr view ${{ steps.pr-create.outputs.pull-request-number }} --json headRefName --template '{{.headRefName}}')
+          echo "sha=$(git rev-parse origin/$HEADREF)" >> $GITHUB_OUTPUT
+          echo "headref=$HEADREF" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Request dispatched PR build
+        if: ${{ steps.pr-create.outputs.pull-request-number && steps.pr-create.outputs.pull-request-operation != 'closed' }}
+        run: >
+          gh workflow run -R Graylog2/graylog-project-internal pr-build.yml --ref master
+          -f caller_repo=${{ github.repository }}
+          -f caller_pr_nr=${{ steps.pr-create.outputs.pull-request-number }}
+          -f caller_base_branch=master
+          -f caller_head_branch=${{ steps.pr-infos.outputs.headref }}
+          -f head_sha=${{ steps.pr-infos.outputs.sha }}
+          -f initial_actor="Dr. Lint-a-lot"
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT_GRAYLOG_PROJECT_INTERNAL_WORKFLOW_RW }}

.github/workflows/updating-lockfile.yml

@@ -2,7 +2,7 @@ name: Updating yarn lockfile
 
 on:
   schedule:
-    - cron: '0 */12 * * *'
+    - cron: '0 */12 * * MON-FRI'
 
   workflow_dispatch:
 
@@ -18,7 +18,8 @@ jobs:
       - name: Install dependencies
         run: yarn install
       - name: Create/Update Pull Request
-        uses: Graylog2/create-pull-request@7380612b49221684fefa025244f2ef4008ae50ad
+        id: pr-create
+        uses: Graylog2/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38
         with:
           title: Updating yarn lockfile
           body: This PR was created by a job that is running periodically to update the yarn lockfile after transitive dependencies have been updated.
@@ -27,3 +28,26 @@ jobs:
           committer: Gary Bot <[email protected]>
           commit-message: Updating yarn lockfile
           delete-branch: true
+
+      - name: Get headRef and SHA of PR
+        id: pr-infos
+        if: ${{ steps.pr-create.outputs.pull-request-number && steps.pr-create.outputs.pull-request-operation != 'closed' }}
+        run: |
+          HEADREF=$(gh pr view ${{ steps.pr-create.outputs.pull-request-number }} --json headRefName --template '{{.headRefName}}')
+          echo "sha=$(git rev-parse origin/$HEADREF)" >> $GITHUB_OUTPUT
+          echo "headref=$HEADREF" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Request dispatched PR build
+        if: ${{ steps.pr-create.outputs.pull-request-number && steps.pr-create.outputs.pull-request-operation != 'closed' }}
+        run: >
+          gh workflow run -R Graylog2/graylog-project-internal pr-build.yml --ref master
+          -f caller_repo=${{ github.repository }}
+          -f caller_pr_nr=${{ steps.pr-create.outputs.pull-request-number }}
+          -f caller_base_branch=master
+          -f caller_head_branch=${{ steps.pr-infos.outputs.headref }}
+          -f head_sha=${{ steps.pr-infos.outputs.sha }}
+          -f initial_actor="Dr. Lint-a-lot"
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT_GRAYLOG_PROJECT_INTERNAL_WORKFLOW_RW }}

UPGRADING.md

@@ -42,15 +42,47 @@ message parsing for these inputs is expected to be released in an upcoming relea
 ### AWS Security Lake input
 
 Changed fields:
-- `message`: Now contains the full JSON content of the log message. The `vendor_event_description` field now contains the previous `message` field value for backwards-compatibility. 
+- `message`: Now contains the full JSON content of the log message. 
 - The message `timestamp` field is now set to the current Graylog system date/time, instead of the previously used log `time` value. The `event_created` field now contains the previous `time` value for backwards-compatibility.
 
 Added fields:
 - `event_created`: Contains the `time` log value.
 - `event_source_input`: Contains the static value `aws_security_lake`.
-- `vendor_event_description`: Contains the value which was previously present in the `message` log field.
+- `vendor_event_type`: Contains the `type_name` log value (previously in the `event_log_name` field).
+- `vendor_event_severity`: Contains the `severity` log value (previously in the `event_severity` field).
 - `vendor_version`: Contains the `metadata.product.version` log value.
 
+Removed fields:
+- `answers`
+- `api`
+- `class_name` (this value is still available in the `source` field)
+- `cloud`
+- `compliance`
+- `confidence`
+- `connection_info`
+- `destination_ip`
+- `destination_port`
+- `destination_subnet_id`
+- `destination_vpc_id`
+- `event_action,`
+- `event_end`
+- `event_log_name`
+- `event_severity`
+- `event_start`
+- `finding`
+- `http_request`
+- `identity`
+- `malware`
+- `process`
+- `query`
+- `rcode`
+- `source_ip`
+- `source_port`
+- `source_subnet_id`
+- `source_vpc_id`
+- `traffic`
+- `vulnerabilities`
+
 ### Office 365 input
 
 Changed fields:

changelog/unreleased/issue-17523.toml

@@ -0,0 +1,5 @@
+type = "c"
+message = "Ensure password secret meets the minimum length requirement if using/for the DataNode."
+
+issues = ["17523"]
+pulls = ["17719"]

changelog/unreleased/issue-17647.toml

@@ -0,0 +1,5 @@
+type = "c"
+message = "Updated the select_jsonpath pipeline function to accept JSON strings as the `json` parameter in addition to parsed JsonNode objects."
+
+issues = ["17647"]
+pulls = ["17683"]

changelog/unreleased/issue-7629.toml

@@ -0,0 +1,5 @@
+type = "added"
+message = "Add Create new dashboard widget action"
+
+issues = ["7629"]
+pulls = ["17582"]

changelog/unreleased/pr-17601.toml

@@ -0,0 +1,7 @@
+type = "c"
+message = "Changed logging in Message class. When invalid message key is found and ignored, that fact is logged with INFO level. Rate limited log is used in order to not overwhelm logs with this kind of log messages."
+
+issues = [""]
+pulls = ["17601"]
+
+

changelog/unreleased/pr-17642.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Fix distribution of table column width for print version of data table and message list widget"
+
+issues = ["Graylog2/graylog-plugin-enterprise#6158"]
+pulls = ["17642"]

changelog/unreleased/pr-17656.toml

@@ -0,0 +1,4 @@
+type = "f"
+message = "Remove entity from bulk select state, when removing single entity in tables with bulk actions."
+
+pulls = ["17656"]

changelog/unreleased/pr-17678.toml

@@ -0,0 +1,6 @@
+type = "f"
+message = "Fixing text alignment of highlighted numbers in data tables."
+
+pulls = ["17678"]
+
+

changelog/unreleased/pr-17693.toml

@@ -0,0 +1,4 @@
+type = "c"
+message = "Unify the way numeric fields are displayed in the message table widget, compared with the data table widget."
+
+pulls = ["17693"]

data-node/src/main/java/org/graylog/datanode/Configuration.java

@@ -288,8 +288,8 @@ public Duration getIndexerJwtAuthTokenExpirationDuration() {
     @ValidatorMethod
     @SuppressWarnings("unused")
     public void validatePasswordSecret() throws ValidationException {
-        if (passwordSecret == null || passwordSecret.length() < 16) {
-            throw new ValidationException("The minimum length for \"password_secret\" is 16 characters.");
+        if (passwordSecret == null || passwordSecret.length() < 64) {
+            throw new ValidationException("The minimum length for \"password_secret\" is 64 characters.");
         }
     }
 

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java

@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.pipelineprocessor.functions.json;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
@@ -44,18 +45,20 @@ public class SelectJsonPath extends AbstractFunction<Map<String, Object>> {
 
     public static final String NAME = "select_jsonpath";
 
+    private final ObjectMapper objectMapper;
     private final Configuration configuration;
-    private final ParameterDescriptor<JsonNode, JsonNode> jsonParam;
+    private final ParameterDescriptor<Object, Object> jsonParam;
     private final ParameterDescriptor<Map<String, String>, Map<String, JsonPath>> pathsParam;
 
     @Inject
     public SelectJsonPath(ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
         configuration = Configuration.builder()
                 .options(Option.SUPPRESS_EXCEPTIONS)
                 .jsonProvider(new JacksonJsonNodeJsonProvider(objectMapper))
                 .build();
 
-        jsonParam = ParameterDescriptor.type("json", JsonNode.class).description("A parsed JSON tree").build();
+        jsonParam = ParameterDescriptor.type("json", Object.class).description("A parsed JSON tree or String representation of a JSON tree").build();
         // sigh generics and type erasure
         //noinspection unchecked
         pathsParam = ParameterDescriptor.type("paths",
@@ -70,7 +73,21 @@ public SelectJsonPath(ObjectMapper objectMapper) {
 
     @Override
     public Map<String, Object> evaluate(FunctionArgs args, EvaluationContext context) {
-        final JsonNode json = jsonParam.required(args, context);
+        final Object jsonObj = jsonParam.required(args, context);
+        JsonNode json = null;
+        if (jsonObj instanceof JsonNode jsonNode) {
+            json = jsonNode;
+        } else if (jsonObj instanceof String jsonString) {
+            try {
+                json = objectMapper.readTree(jsonString);
+            } catch (JsonProcessingException e) {
+                log.warn(context.pipelineErrorMessage("Unable to parse JSON"), e);
+            }
+        } else {
+            throw new IllegalArgumentException(context.pipelineErrorMessage(
+                    "`json` parameter must be a parsed JSON tree or String representation of a JSON tree"));
+        }
+
         final Map<String, JsonPath> paths = pathsParam.required(args, context);
         if (json == null || paths == null) {
             return Collections.emptyMap();

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java

@@ -48,6 +48,7 @@
 import org.graylog2.plugin.Messages;
 import org.graylog2.plugin.messageprocessors.MessageProcessor;
 import org.graylog2.plugin.streams.Stream;
+import org.graylog2.plugin.utilities.ratelimitedlog.RateLimitedLogFactory;
 import org.graylog2.shared.buffers.processors.ProcessBufferProcessor;
 import org.graylog2.shared.messageq.MessageQueueAcknowledger;
 import org.graylog2.shared.metrics.MetricUtils;
@@ -470,12 +471,8 @@ public String className() {
         }
     }
 
-    public static RateLimitedLog getRateLimitedLog(Class clazz) {
-        final Logger baseLog = LoggerFactory.getLogger(clazz);
-        return RateLimitedLog
-                .withRateLimit(baseLog)
-                .maxRate(5).every(Duration.ofSeconds(10))
-                .build();
+    public static RateLimitedLog getRateLimitedLog(final Class<?> clazz) {
+        return RateLimitedLogFactory.createRateLimitedLog(clazz, 5, Duration.ofSeconds(10));
     }
 
     public static class State {

graylog2-server/src/main/java/org/graylog2/audit/AuditEventTypes.java

@@ -76,6 +76,10 @@ public class AuditEventTypes implements PluginAuditEventTypes {
     public static final String EXTRACTOR_ORDER_UPDATE = PREFIX + "extractor_order:update";
     public static final String FIELD_TYPE_MAPPING_CREATE = PREFIX + "field_type_mapping:create";
     public static final String FIELD_TYPE_MAPPING_DELETE = PREFIX + "field_type_mapping:delete";
+
+    public static final String INDEX_FIELD_TYPE_PROFILE_CREATE = PREFIX + "index_field_type_profile:create";
+    public static final String INDEX_FIELD_TYPE_PROFILE_UPDATE = PREFIX + "index_field_type_profile:update";
+    public static final String INDEX_FIELD_TYPE_PROFILE_DELETE = PREFIX + "index_field_type_profile:delete";
     public static final String EXTRACTOR_UPDATE = PREFIX + "extractor:update";
     public static final String GETTING_STARTED_GUIDE_OPT_OUT_CREATE = PREFIX + "getting_started_guide_opt_out:create";
     public static final String GRANTS_UPDATE = PREFIX + "grants:update";
@@ -223,6 +227,9 @@ public class AuditEventTypes implements PluginAuditEventTypes {
             .add(GROK_PATTERN_DELETE)
             .add(GROK_PATTERN_IMPORT_CREATE)
             .add(GROK_PATTERN_UPDATE)
+            .add(INDEX_FIELD_TYPE_PROFILE_CREATE)
+            .add(INDEX_FIELD_TYPE_PROFILE_DELETE)
+            .add(INDEX_FIELD_TYPE_PROFILE_UPDATE)
             .add(INDEX_SET_CREATE)
             .add(INDEX_SET_DELETE)
             .add(INDEX_SET_UPDATE)

graylog2-server/src/main/java/org/graylog2/indexer/indexset/profile/IndexFieldTypeProfile.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.indexer.indexset.profile;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.graylog2.indexer.indexset.CustomFieldMappings;
+import org.mongojack.Id;
+import org.mongojack.ObjectId;
+
+import javax.annotation.Nullable;
+
+public record IndexFieldTypeProfile(@JsonProperty(ID_FIELD_NAME) @Nullable @Id @ObjectId String id,
+                                    @JsonProperty(NAME_FIELD_NAME) String name,
+                                    @JsonProperty(DESCRIPTION_FIELD_NAME) String description,
+                                    @JsonProperty(CUSTOM_MAPPINGS_FIELD_NAME) CustomFieldMappings customFieldMappings) {
+
+    public static final String ID_FIELD_NAME = "id";
+    public static final String NAME_FIELD_NAME = "name";
+    public static final String DESCRIPTION_FIELD_NAME = "description";
+    public static final String CUSTOM_MAPPINGS_FIELD_NAME = "custom_field_mappings";
+
+    public IndexFieldTypeProfile(final IndexFieldTypeProfileData data) {
+        this(null, data.name(), data.description(), data.customFieldMappings());
+    }
+}