Merge branch 'master' into fix/npe-when-replaying-security-event-search

Graylog2 · Dec 23, 2024 · f11ed82 · f11ed82
2 parents 23ae9a2 + 3981080
commit f11ed82
Show file tree

Hide file tree

Showing 27 changed files with 342 additions and 93 deletions.
diff --git a/changelog/unreleased/pr-21197.toml b/changelog/unreleased/pr-21197.toml
@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Add archive restore retry on mapper parsing exception."
+
+issues = ["graylog-plugin-enterprise#9208"]
+pulls = ["21197", "Graylog2/graylog-plugin-enterprise#9413"]
diff --git a/changelog/unreleased/pr-21206.toml b/changelog/unreleased/pr-21206.toml
@@ -0,0 +1,5 @@
+type = "f"
+message = "Filters system event definitions from list of entities that can be exported in a content pack."
+
+pulls = ["21206"]
+issues = ["21166"]
diff --git a/changelog/unreleased/pr-21217.toml b/changelog/unreleased/pr-21217.toml
@@ -0,0 +1,4 @@
+type = "f"
+message = "Fixed bug where inputs with multiple encrypted configuration values could not be saved."
+
+pulls = ["21217"]
diff --git a/...-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ElasticsearchClient.java b/...-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ElasticsearchClient.java
@@ -42,6 +42,7 @@
 import org.graylog2.indexer.BatchSizeTooLargeException;
 import org.graylog2.indexer.IndexNotFoundException;
 import org.graylog2.indexer.InvalidWriteTargetException;
+import org.graylog2.indexer.MapperParsingException;
 import org.graylog2.indexer.MasterNotDiscoveredException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -202,6 +203,9 @@ private ElasticsearchException exceptionFrom(Exception e, String errorMessage) {
             if (isBatchSizeTooLargeException(elasticsearchException)) {
                 throw new BatchSizeTooLargeException(elasticsearchException.getMessage());
             }
+            if (isMapperParsingExceptionException(elasticsearchException)) {
+                throw new MapperParsingException(elasticsearchException.getMessage());
+            }
         } else if (e instanceof IOException && e.getCause() instanceof ContentTooLongException) {
             throw new BatchSizeTooLargeException(e.getMessage());
         }
@@ -231,6 +235,10 @@ private boolean isIndexNotFoundException(ElasticsearchException elasticsearchExc
         return elasticsearchException.getMessage().contains("index_not_found_exception");
     }
 
+    private boolean isMapperParsingExceptionException(ElasticsearchException openSearchException) {
+        return openSearchException.getMessage().contains("mapper_parsing_exception");
+    }
+
     private boolean isBatchSizeTooLargeException(ElasticsearchException elasticsearchException) {
         if (elasticsearchException instanceof ElasticsearchStatusException statusException) {
             if (statusException.getCause() instanceof ResponseException responseException) {

diff --git a/...g-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/OpenSearchClient.java b/...g-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/OpenSearchClient.java
@@ -42,6 +42,7 @@
 import org.graylog2.indexer.BatchSizeTooLargeException;
 import org.graylog2.indexer.IndexNotFoundException;
 import org.graylog2.indexer.InvalidWriteTargetException;
+import org.graylog2.indexer.MapperParsingException;
 import org.graylog2.indexer.MasterNotDiscoveredException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -202,6 +203,9 @@ private OpenSearchException exceptionFrom(Exception e, String errorMessage) {
             if (isBatchSizeTooLargeException(openSearchException)) {
                 throw new BatchSizeTooLargeException(openSearchException.getMessage());
             }
+            if (isMapperParsingExceptionException(openSearchException)) {
+                throw new MapperParsingException(openSearchException.getMessage());
+            }
         } else if (e instanceof IOException && e.getCause() instanceof ContentTooLongException) {
             throw new BatchSizeTooLargeException(e.getMessage());
         }
@@ -231,6 +235,10 @@ private boolean isIndexNotFoundException(OpenSearchException openSearchException
         return openSearchException.getMessage().contains("index_not_found_exception");
     }
 
+    private boolean isMapperParsingExceptionException(OpenSearchException openSearchException) {
+        return openSearchException.getMessage().contains("mapper_parsing_exception");
+    }
+
     private boolean isBatchSizeTooLargeException(OpenSearchException openSearchException) {
         if (openSearchException instanceof OpenSearchStatusException statusException) {
             if (statusException.getCause() instanceof ResponseException responseException) {

diff --git a/...g/graylog/events/processor/systemnotification/SystemNotificationEventProcessorConfig.java b/...g/graylog/events/processor/systemnotification/SystemNotificationEventProcessorConfig.java
@@ -62,6 +62,11 @@ public EventProcessorConfigEntity toContentPackEntity(EntityDescriptorIds entity
         return null;
     }
 
+    @Override
+    public boolean isContentPackExportable() {
+        return false;
+    }
+
     @Override
     public void resolveNativeEntity(EntityDescriptor entityDescriptor, MutableGraph<EntityDescriptor> mutableGraph) {
     }

diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/MapperParsingException.java b/graylog2-server/src/main/java/org/graylog2/indexer/MapperParsingException.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.indexer;
+
+public class MapperParsingException extends ElasticsearchException {
+    public MapperParsingException(String errorMessage) {
+        super(errorMessage);
+    }
+}
diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java
@@ -36,6 +36,7 @@
 import org.graylog2.indexer.IndexNotFoundException;
 import org.graylog2.indexer.IndexSet;
 import org.graylog2.indexer.IndexTemplateNotFoundException;
+import org.graylog2.indexer.MapperParsingException;
 import org.graylog2.indexer.indexset.CustomFieldMappings;
 import org.graylog2.indexer.indexset.IndexSetConfig;
 import org.graylog2.indexer.indexset.IndexSetMappingTemplate;
@@ -227,7 +228,7 @@ public void deleteIndexTemplate(IndexSet indexSet) {
     }
 
     public boolean create(String indexName, IndexSet indexSet) {
-        return create(indexName, indexSet, null, null );
+        return create(indexName, indexSet, null, null);
     }
 
     public boolean create(String indexName,
@@ -248,6 +249,10 @@ public boolean create(String indexName,
 
             indicesAdapter.create(indexName, settings, mappings);
         } catch (Exception e) {
+            if ((indexSettings != null || indexMapping != null) && e instanceof MapperParsingException) {
+                LOG.info("Couldn't create index {}. Error: {}. Fall back to default settings/mappings and retry.", indexName, e.getMessage(), e);
+                return create(indexName, indexSet, null, null);
+            }
             LOG.warn("Couldn't create index {}. Error: {}", indexName, e.getMessage(), e);
             auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_CREATE, ImmutableMap.of("indexName", indexName));
             return false;
@@ -259,7 +264,7 @@ public boolean create(String indexName,
     private Optional<IndexMappingTemplate> indexMapping(IndexSet indexSet) {
         try {
             return Optional.of(indexMappingFactory.createIndexMapping(indexSet.getConfig()));
-        }catch (IgnoreIndexTemplate e){
+        } catch (IgnoreIndexTemplate e) {
             return Optional.empty();
         }
     }

diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/InputServiceImpl.java b/graylog2-server/src/main/java/org/graylog2/inputs/InputServiceImpl.java
@@ -544,7 +544,6 @@ protected void fieldTransformations(Map<String, Object> doc) {
         for (Map.Entry<String, Object> x : doc.entrySet()) {
             if (x.getValue() instanceof EncryptedValue encryptedValue) {
                 doc.put(x.getKey(), objectMapper.convertValue(encryptedValue, TypeReferences.MAP_STRING_OBJECT));
-                return;
             }
         }
         super.fieldTransformations(doc);

diff --git a/graylog2-server/src/test/java/org/graylog2/inputs/InputServiceImplTest.java b/graylog2-server/src/test/java/org/graylog2/inputs/InputServiceImplTest.java
@@ -164,30 +164,39 @@ public void handlesEncryptedValue() throws ValidationException, NotFoundExceptio
         final MessageInput.Config inputConfig = mock(MessageInput.Config.class);
         when(inputConfig.combinedRequestedConfiguration()).thenReturn(ConfigurationRequest.createWithFields(
                 new TextField("encrypted", "", "", "",
+                        ConfigurationField.Optional.OPTIONAL, true),
+                new TextField("encrypted2", "", "", "",
                         ConfigurationField.Optional.OPTIONAL, true)
         ));
         when(messageInputFactory.getConfig("test type")).thenReturn(Optional.of(
                 inputConfig
         ));
 
         final EncryptedValue secret = encryptedValueService.encrypt("secret");
+        final EncryptedValue secret2 = encryptedValueService.encrypt("secret2");
         final String id = inputService.save(new InputImpl(Map.of(
                 FIELD_TYPE, "test type",
                 FIELD_TITLE, "test title",
                 FIELD_CREATED_AT, new Date(),
                 FIELD_CREATOR_USER_ID, "test creator",
                 FIELD_CONFIGURATION, Map.of(
-                        "encrypted", secret
+                        "encrypted", secret,
+                        "encrypted2", secret2
                 )
         )));
 
         assertThat(id).isNotBlank();
 
-        assertThat(inputService.find(id)).satisfies(input ->
-                assertThat(input.getConfiguration()).hasEntrySatisfying("encrypted", value -> {
-                    assertThat(value).isInstanceOf(EncryptedValue.class);
-                    assertThat(value).isEqualTo(secret);
-                }));
+        assertThat(inputService.find(id)).satisfies(input -> {
+            assertThat(input.getConfiguration()).hasEntrySatisfying("encrypted", value -> {
+                assertThat(value).isInstanceOf(EncryptedValue.class);
+                assertThat(value).isEqualTo(secret);
+            });
+            assertThat(input.getConfiguration()).hasEntrySatisfying("encrypted2", value -> {
+                assertThat(value).isInstanceOf(EncryptedValue.class);
+                assertThat(value).isEqualTo(secret2);
+            });
+        });
 
         assertThat(inputService.allByType("test type")).hasSize(1).first().satisfies(input ->
                 assertThat(input.getConfiguration()).hasEntrySatisfying("encrypted", value -> {

diff --git a/graylog2-web-interface/packages/eslint-config-graylog/package.json b/graylog2-web-interface/packages/eslint-config-graylog/package.json
@@ -23,7 +23,7 @@
     "eslint-plugin-compat": "4.2.0",
     "eslint-plugin-graylog": "file:../eslint-plugin-graylog",
     "eslint-plugin-import": "2.25.3",
-    "eslint-plugin-jest": "28.9.0",
+    "eslint-plugin-jest": "28.10.0",
     "eslint-plugin-jest-dom": "5.5.0",
     "eslint-plugin-jest-formatting": "3.1.0",
     "eslint-plugin-jsx-a11y": "6.10.2",

diff --git a/graylog2-web-interface/src/components/common/EntityFilters/EntityFilters.test.tsx b/graylog2-web-interface/src/components/common/EntityFilters/EntityFilters.test.tsx
@@ -19,11 +19,13 @@ import { render, screen, waitFor, within } from 'wrappedTestingLibrary';
 import userEvent from '@testing-library/user-event';
 import { OrderedMap } from 'immutable';
 import type { Optional } from 'utility-types';
+import { Formik, Form } from 'formik';
 
-import type { Attributes } from 'stores/PaginationTypes';
+import type { Attributes, FilterComponentProps } from 'stores/PaginationTypes';
 import { asMock } from 'helpers/mocking';
 import useFilterValueSuggestions from 'components/common/EntityFilters/hooks/useFilterValueSuggestions';
 import useFiltersWithTitle from 'components/common/EntityFilters/hooks/useFiltersWithTitle';
+import { ModalSubmit, FormikInput } from 'components/common';
 
 import OriginalEntityFilters from './EntityFilters';
 
@@ -36,10 +38,31 @@ jest.mock('logic/generateId', () => jest.fn(() => 'filter-id'));
 jest.mock('components/common/EntityFilters/hooks/useFilterValueSuggestions');
 jest.mock('components/common/EntityFilters/hooks/useFiltersWithTitle');
 
+const CustomFilterInput = ({ filter, onSubmit }: FilterComponentProps) => (
+  <div data-testid="custom-component-form">
+    <Formik initialValues={{ value: filter?.value }} onSubmit={({ value }) => onSubmit({ title: value, value })}>
+      {({ isValid }) => (
+        <Form>
+          <FormikInput type="text"
+                       id="custom-input"
+                       name="value"
+                       formGroupClassName=""
+                       required
+                       placeholder="My custom input" />
+          <ModalSubmit submitButtonText={`${filter ? 'Update' : 'Create'} filter`}
+                       bsSize="small"
+                       disabledSubmit={!isValid}
+                       displayCancel={false} />
+        </Form>
+      )}
+    </Formik>
+  </div>
+);
+
 describe('<EntityFilters />', () => {
   const onChangeFiltersWithTitle = jest.fn();
   const setUrlQueryFilters = jest.fn();
-  const attributes = [
+  const attributes: Attributes = [
     { id: 'title', title: 'Title', sortable: true },
     { id: 'description', title: 'Description', sortable: true },
     {
@@ -88,7 +111,14 @@ describe('<EntityFilters />', () => {
       title: 'Generic Attribute',
       type: 'STRING',
     },
-  ] as Attributes;
+    {
+      id: 'customComponent',
+      filterable: true,
+      title: 'Custom Component Attribute',
+      type: 'STRING',
+      filter_component: CustomFilterInput,
+    },
+  ];
 
   const EntityFilters = (props: Optional<React.ComponentProps<typeof OriginalEntityFilters>, 'setUrlQueryFilters' | 'attributes'>) => (
     <OriginalEntityFilters setUrlQueryFilters={setUrlQueryFilters} attributes={attributes} {...props} />
@@ -424,6 +454,52 @@ describe('<EntityFilters />', () => {
     });
   });
 
+  describe('custom component attribute', () => {
+    it('provides text input to create filter', async () => {
+      render(
+        <EntityFilters urlQueryFilters={OrderedMap()} />,
+      );
+
+      userEvent.click(await screen.findByRole('button', { name: /create filter/i }));
+
+      userEvent.click(await screen.findByRole('menuitem', { name: /custom component/i }));
+
+      const filterInput = await screen.findByPlaceholderText('My custom input');
+      userEvent.type(filterInput, 'foo');
+
+      const form = await screen.findByTestId('custom-component-form');
+      userEvent.click(await within(form).findByRole('button', { name: /create filter/i }));
+
+      await waitFor(() => {
+        expect(setUrlQueryFilters).toHaveBeenCalledWith(OrderedMap({ customComponent: ['foo'] }));
+      });
+    });
+
+    it('allows changing filter', async () => {
+      asMock(useFiltersWithTitle).mockReturnValue({
+        data: OrderedMap({ customComponent: [{ title: 'foo', value: 'foo' }] }),
+        onChange: onChangeFiltersWithTitle,
+        isInitialLoading: false,
+      });
+
+      render(
+        <EntityFilters urlQueryFilters={OrderedMap()} />,
+      );
+
+      userEvent.click(await screen.findByText('foo'));
+
+      const filterInput = await screen.findByPlaceholderText('My custom input');
+      userEvent.type(filterInput, '{selectall}bar');
+
+      const form = await screen.findByTestId('custom-component-form');
+      userEvent.click(await within(form).findByRole('button', { name: /update filter/i }));
+
+      await waitFor(() => {
+        expect(setUrlQueryFilters).toHaveBeenCalledWith(OrderedMap({ customComponent: ['bar'] }));
+      });
+    });
+  });
+
   it('should display active filters', async () => {
     asMock(useFiltersWithTitle).mockReturnValue({
       data: OrderedMap({

diff --git a/...interface/src/components/common/EntityFilters/FilterConfiguration/FilterConfiguration.tsx b/...interface/src/components/common/EntityFilters/FilterConfiguration/FilterConfiguration.tsx
@@ -16,28 +16,31 @@
  */
 import * as React from 'react';
 
-import type { Attribute } from 'stores/PaginationTypes';
-import type { Filters, Filter } from 'components/common/EntityFilters/types';
+import type { FilterComponentProps } from 'stores/PaginationTypes';
 import { MenuItem } from 'components/bootstrap';
 import {
   isAttributeWithFilterOptions,
-  isAttributeWithRelatedCollection, isDateAttribute,
+  isAttributeWithRelatedCollection, isDateAttribute, isCustomComponentFilter,
 } from 'components/common/EntityFilters/helpers/AttributeIdentification';
-import GenericFilterInput from 'components/common/EntityFilters/FilterConfiguration/GenericFilterInput';
 
+import SuggestionsListFilter from './SuggestionsListFilter';
+import GenericFilterInput from './GenericFilterInput';
 import StaticOptionsList from './StaticOptionsList';
-import SuggestionsList from './SuggestionsList';
 import DateRangeForm from './DateRangeForm';
 
-type Props = {
-  attribute: Attribute,
-  filter?: Filter,
-  filterValueRenderer: (value: Filter['value'], title: string) => React.ReactNode | undefined,
-  onSubmit: (filter: { title: string, value: string }, closeDropdown?: boolean) => void,
-  allActiveFilters: Filters | undefined,
-}
+const FilterComponent = ({ allActiveFilters, attribute, filter = undefined, filterValueRenderer, onSubmit }: FilterComponentProps) => {
+  if (isCustomComponentFilter(attribute)) {
+    const CustomFilterComponent = attribute.filter_component;
+
+    return (
+      <CustomFilterComponent attribute={attribute}
+                             filterValueRenderer={filterValueRenderer}
+                             onSubmit={onSubmit}
+                             allActiveFilters={allActiveFilters}
+                             filter={filter} />
+    );
+  }
 
-const FilterComponent = ({ allActiveFilters, attribute, filter, filterValueRenderer, onSubmit }: Pick<Props, 'allActiveFilters' | 'attribute' | 'filter' | 'filterValueRenderer' | 'onSubmit'>) => {
   if (isAttributeWithFilterOptions(attribute)) {
     return (
       <StaticOptionsList attribute={attribute}
@@ -49,11 +52,11 @@ const FilterComponent = ({ allActiveFilters, attribute, filter, filterValueRende
 
   if (isAttributeWithRelatedCollection(attribute)) {
     return (
-      <SuggestionsList attribute={attribute}
-                       filterValueRenderer={filterValueRenderer}
-                       onSubmit={onSubmit}
-                       allActiveFilters={allActiveFilters}
-                       filter={filter} />
+      <SuggestionsListFilter attribute={attribute}
+                             filterValueRenderer={filterValueRenderer}
+                             onSubmit={onSubmit}
+                             allActiveFilters={allActiveFilters}
+                             filter={filter} />
     );
   }
 
@@ -73,7 +76,7 @@ export const FilterConfiguration = ({
   filter = undefined,
   filterValueRenderer,
   onSubmit,
-}: Props) => (
+}: FilterComponentProps) => (
   <>
     <MenuItem header>{filter ? 'Edit' : 'Create'} {attribute.title.toLowerCase()} filter</MenuItem>
     <FilterComponent attribute={attribute}