From f197bbb9dc6fad5916c6734c7bebf2c4e9699255 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Tue, 17 Dec 2024 13:08:01 +0100 Subject: [PATCH 01/12] Code cleanup (#21177) Co-authored-by: Matthias Oesterheld <33032967+moesterheld@users.noreply.github.com> --- .../graylog/datanode/configuration/DatanodeConfiguration.java | 2 -- .../datanode/configuration/DatanodeConfigurationProvider.java | 1 - .../beans/impl/OpensearchCommonConfigurationBean.java | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfiguration.java b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfiguration.java index 4a2ef38e0605..3b78df4747ea 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfiguration.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfiguration.java @@ -16,7 +16,6 @@ */ package org.graylog.datanode.configuration; -import org.graylog.datanode.OpensearchDistribution; import org.graylog2.security.IndexerJwtAuthTokenProvider; /** @@ -27,7 +26,6 @@ public record DatanodeConfiguration( OpensearchDistributionProvider opensearchDistributionProvider, DatanodeDirectories datanodeDirectories, int processLogsBufferSize, - String opensearchHeap, IndexerJwtAuthTokenProvider indexerJwtAuthTokenProvider ) { } diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfigurationProvider.java b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfigurationProvider.java index ae4f7a606179..3cfc53f0d411 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfigurationProvider.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfigurationProvider.java @@ -40,7 +40,6 @@ public DatanodeConfigurationProvider( opensearchDistributionProvider, DatanodeDirectories.fromConfiguration(localConfiguration, nodeId), localConfiguration.getProcessLogsBufferSize(), - localConfiguration.getOpensearchHeap(), jwtTokenProvider ); } diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java index 6705e69244fa..a51bb0a1112d 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java @@ -43,8 +43,8 @@ public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationP return DatanodeConfigurationPart.builder() .properties(commonOpensearchConfig(buildParams)) .nodeRoles(localConfiguration.getNodeRoles()) - .javaOpt("-Xms%s".formatted(datanodeConfiguration.opensearchHeap())) - .javaOpt("-Xmx%s".formatted(datanodeConfiguration.opensearchHeap())) + .javaOpt("-Xms%s".formatted(localConfiguration.getOpensearchHeap())) + .javaOpt("-Xmx%s".formatted(localConfiguration.getOpensearchHeap())) .javaOpt("-Dopensearch.transport.cname_in_publish_address=true") .build(); } From b9a8df381e05e61c2b3315c11920cedf6ddfc62d Mon Sep 17 00:00:00 2001 From: Anton Ebel Date: Tue, 17 Dec 2024 15:01:17 +0100 Subject: [PATCH 02/12] add javadoc for decodeSafe method (#21207) --- .../main/java/org/graylog2/plugin/inputs/codecs/Codec.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java index b99f13a94cae..0f2591b35228 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java @@ -28,6 +28,12 @@ public interface Codec { + /** + * @param rawMessage + * @return an empty Optional if RawMessage contains payload data that should not be decoded. + * Otherwise, an Optional with a Message that can be processed further. + * @throws org.graylog2.plugin.inputs.failure.InputProcessingException if an error occurs during decoding. + */ default Optional decodeSafe(@Nonnull RawMessage rawMessage) { return Optional.ofNullable(decode(rawMessage)); } From f4a129171ad40ecaa63bd89768d3e7608f2c9950 Mon Sep 17 00:00:00 2001 From: Linus Pahl <46300478+linuspahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:25:43 +0100 Subject: [PATCH 03/12] Create `ClipboardIconButton`. (#21135) * Create `ClipboardIconButton`. * Add tests. * Define fixed icon name. * Use regular icon type. * Update test * Fix linter hints --- .../common/ClipboardButton.test.tsx | 38 +++++++++++ .../src/components/common/ClipboardButton.tsx | 68 ++++++------------- .../components/common/ClipboardContainer.tsx | 63 +++++++++++++++++ .../common/ClipboardIconButton.test.tsx | 38 +++++++++++ .../components/common/ClipboardIconButton.tsx | 51 ++++++++++++++ .../src/components/common/index.tsx | 1 + 6 files changed, 212 insertions(+), 47 deletions(-) create mode 100644 graylog2-web-interface/src/components/common/ClipboardButton.test.tsx create mode 100644 graylog2-web-interface/src/components/common/ClipboardContainer.tsx create mode 100644 graylog2-web-interface/src/components/common/ClipboardIconButton.test.tsx create mode 100644 graylog2-web-interface/src/components/common/ClipboardIconButton.tsx diff --git a/graylog2-web-interface/src/components/common/ClipboardButton.test.tsx b/graylog2-web-interface/src/components/common/ClipboardButton.test.tsx new file mode 100644 index 000000000000..fa1d62cc31e4 --- /dev/null +++ b/graylog2-web-interface/src/components/common/ClipboardButton.test.tsx @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import React from 'react'; +import { render, screen } from 'wrappedTestingLibrary'; +import userEvent from '@testing-library/user-event'; + +import copyToClipboard from 'util/copyToClipboard'; + +import ClipboardButton from './ClipboardButton'; + +jest.mock('util/copyToClipboard', () => jest.fn(() => Promise.resolve())); + +describe('ClipboardButton', () => { + it('should copy provided text to clipboard', async () => { + const text = 'Text to copy'; + render(); + + userEvent.click(await screen.findByRole('button', { name: /click here to copy/i })); + + expect(copyToClipboard).toHaveBeenCalledWith(text); + + await screen.findByText('Copied!'); + }); +}); diff --git a/graylog2-web-interface/src/components/common/ClipboardButton.tsx b/graylog2-web-interface/src/components/common/ClipboardButton.tsx index 6c184726247d..cbaa7609d04a 100644 --- a/graylog2-web-interface/src/components/common/ClipboardButton.tsx +++ b/graylog2-web-interface/src/components/common/ClipboardButton.tsx @@ -15,14 +15,11 @@ * . */ import * as React from 'react'; -import { useCallback, useState } from 'react'; -import { useTimeout } from '@mantine/hooks'; import { Button } from 'components/bootstrap'; import type { BsSize } from 'components/bootstrap/types'; import type { StyleProps } from 'components/bootstrap/Button'; -import copyToClipboard from 'util/copyToClipboard'; -import Tooltip from 'components/common/Tooltip'; +import ClipboardContainer from 'components/common/ClipboardContainer'; /** * Component that renders a button to copy some text in the clipboard when pressed. @@ -40,48 +37,25 @@ type Props = { title: React.ReactNode, } -type Args = { - copied: boolean, - copy: () => void, -} -type CopyButtonProps = { - value: string, - timeout: number, - children: (args: Args) => React.ReactElement, -}; - -const CopyButton = ({ children, value, timeout }: CopyButtonProps) => { - const [copied, setCopied] = useState(false); - const { start } = useTimeout(() => setCopied(false), timeout); - const copy = useCallback(() => copyToClipboard(value).then(() => { setCopied(true); start(); }), [start, value]); - - return children({ copied, copy }); -}; - -const ClipboardButton = ({ bsSize, bsStyle, buttonTitle, className, disabled, onSuccess, text, title }: Props) => { - const button = (copy: () => void) => ( - - ); - - return ( - - {({ copied, copy }) => (copied ? ( - - {button(copy)} - - ) : button(copy))} - - ); -}; +const ClipboardButton = ({ + bsSize = undefined, bsStyle = undefined, buttonTitle = undefined, className = undefined, + disabled = undefined, onSuccess = undefined, text, title, +}: Props) => ( + + {({ copy }) => ( + + )} + +); export default ClipboardButton; diff --git a/graylog2-web-interface/src/components/common/ClipboardContainer.tsx b/graylog2-web-interface/src/components/common/ClipboardContainer.tsx new file mode 100644 index 000000000000..67c08cec4f61 --- /dev/null +++ b/graylog2-web-interface/src/components/common/ClipboardContainer.tsx @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import * as React from 'react'; +import { useCallback, useState } from 'react'; +import { useTimeout } from '@mantine/hooks'; + +import copyToClipboard from 'util/copyToClipboard'; +import Tooltip from 'components/common/Tooltip'; + +/** + * This component can be used as a wrapper for other components. When users click on the children, + * the provided text will be copied to the clipboard, and they see a tooltip for visual feedback. + */ + +type Props = { + children: (props: { copy: () => void }) => JSX.Element, + text: string, +} + +type Args = { + copied: boolean, + copy: () => void, +} + +type CopyProps = { + value: string, + timeout: number, + children: (args: Args) => React.ReactElement, +}; + +const Copy = ({ children, value, timeout }: CopyProps) => { + const [copied, setCopied] = useState(false); + const { start } = useTimeout(() => setCopied(false), timeout); + const copy = useCallback(() => copyToClipboard(value).then(() => { setCopied(true); start(); }), [start, value]); + + return children({ copied, copy }); +}; + +const ClipboardContainer = ({ children, text }: Props) => ( + + {({ copied, copy }) => (copied ? ( + + {children({ copy })} + + ) : children({ copy }))} + +); + +export default ClipboardContainer; diff --git a/graylog2-web-interface/src/components/common/ClipboardIconButton.test.tsx b/graylog2-web-interface/src/components/common/ClipboardIconButton.test.tsx new file mode 100644 index 000000000000..f340b104a3b5 --- /dev/null +++ b/graylog2-web-interface/src/components/common/ClipboardIconButton.test.tsx @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import React from 'react'; +import { render, screen } from 'wrappedTestingLibrary'; +import userEvent from '@testing-library/user-event'; + +import copyToClipboard from 'util/copyToClipboard'; + +import ClipboardIconButton from './ClipboardIconButton'; + +jest.mock('util/copyToClipboard', () => jest.fn(() => Promise.resolve())); + +describe('ClipboardIconButton', () => { + it('should copy provided text to clipboard', async () => { + const text = 'Text to copy'; + render(); + + userEvent.click(await screen.findByRole('button', { name: /click here to copy/i })); + + expect(copyToClipboard).toHaveBeenCalledWith(text); + + await screen.findByText('Copied!'); + }); +}); diff --git a/graylog2-web-interface/src/components/common/ClipboardIconButton.tsx b/graylog2-web-interface/src/components/common/ClipboardIconButton.tsx new file mode 100644 index 000000000000..25992835bfa4 --- /dev/null +++ b/graylog2-web-interface/src/components/common/ClipboardIconButton.tsx @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import * as React from 'react'; + +import ClipboardContainer from 'components/common/ClipboardContainer'; +import { IconButton } from 'components/common'; + +/** + * Component that renders an icon button to copy some text in the clipboard when pressed. + * The text to be copied can be given in the `text` prop, or in an external element through a CSS selector in the `target` prop. + */ + +type Props = { + buttonTitle?: string, + className?: string, + disabled?: boolean, + onSuccess?: () => void, + text: string, +} + +const ClipboardIconButton = ({ buttonTitle = undefined, className = undefined, disabled = undefined, onSuccess = undefined, text }: Props) => ( + + {({ copy }) => ( + { + copy(); + onSuccess?.(); + }} /> + )} + +); + +export default ClipboardIconButton; diff --git a/graylog2-web-interface/src/components/common/index.tsx b/graylog2-web-interface/src/components/common/index.tsx index dcb875a56952..748f36f5c8ad 100644 --- a/graylog2-web-interface/src/components/common/index.tsx +++ b/graylog2-web-interface/src/components/common/index.tsx @@ -29,6 +29,7 @@ export { default as Card } from './Card'; export { default as Center } from './Center'; export { default as Carousel } from './Carousel'; export { default as ClipboardButton } from './ClipboardButton'; +export { default as ClipboardIconButton } from './ClipboardIconButton'; export { default as ColorPicker } from './ColorPicker'; export { default as ColorPickerPopover } from './ColorPickerPopover'; export { default as ConfirmDialog } from './ConfirmDialog'; From b5d905c6f068638b111f13697795879ffb0ef401 Mon Sep 17 00:00:00 2001 From: Linus Pahl <46300478+linuspahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:51:09 +0100 Subject: [PATCH 04/12] Enable defining required permissions for `navigation` web interface plugins. (#21205) * Enable defining permissions for `navigation` web interface plugins. * Adding changelog. --- changelog/unreleased/pr-21205.toml | 8 ++++++++ .../src/@types/graylog-web-plugin/index.d.ts | 1 + 2 files changed, 9 insertions(+) create mode 100644 changelog/unreleased/pr-21205.toml diff --git a/changelog/unreleased/pr-21205.toml b/changelog/unreleased/pr-21205.toml new file mode 100644 index 000000000000..44cd47bce32c --- /dev/null +++ b/changelog/unreleased/pr-21205.toml @@ -0,0 +1,8 @@ +type = "a" +message = "Enable defining required permissions for navigation web interface plugin." + +pulls = ["21205"] +details.user = """ +Before it was only possible to define required permissions for a navigation dropdown item. +""" + diff --git a/graylog2-web-interface/src/@types/graylog-web-plugin/index.d.ts b/graylog2-web-interface/src/@types/graylog-web-plugin/index.d.ts index dd5da4d02769..07551e8da009 100644 --- a/graylog2-web-interface/src/@types/graylog-web-plugin/index.d.ts +++ b/graylog2-web-interface/src/@types/graylog-web-plugin/index.d.ts @@ -56,6 +56,7 @@ type PluginNavigation = { perspective?: string; BadgeComponent?: React.ComponentType<{ text: string }>; position?: 'last' | undefined, + permissions?: string | Array, useIsValidLicense?: () => boolean, } & (PluginNavigationLink | PluginNavigationDropdown) From 7845ab99117250085170ffede89de0e8ae84de7f Mon Sep 17 00:00:00 2001 From: Florian Petersen <188503754+fpetersen-gl@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:21:37 +0100 Subject: [PATCH 05/12] Improve performance for working with indices (#21195) * Issue #18563: Don't iterate over all IndexSets for a given indexName, instead use `getForIndex(String)` directly. Also converted some loops to streaming. * Add changelog * Review comments * Review comments part two --- changelog/unreleased/issue-18563.toml | 5 + .../indexer/MongoIndexSetRegistry.java | 83 ++++++---------- .../indexer/MongoIndexSetRegistryTest.java | 97 +++++++++++++++++++ 3 files changed, 130 insertions(+), 55 deletions(-) create mode 100644 changelog/unreleased/issue-18563.toml diff --git a/changelog/unreleased/issue-18563.toml b/changelog/unreleased/issue-18563.toml new file mode 100644 index 000000000000..7298d46d501d --- /dev/null +++ b/changelog/unreleased/issue-18563.toml @@ -0,0 +1,5 @@ +type = "fixed" +message = "Improve performance of close and delete actions when working with a lot of index sets." + +issues = ["18563"] +pulls = ["21195"] diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/MongoIndexSetRegistry.java b/graylog2-server/src/main/java/org/graylog2/indexer/MongoIndexSetRegistry.java index f601a0aaa717..14a96ed5a560 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/MongoIndexSetRegistry.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/MongoIndexSetRegistry.java @@ -22,15 +22,15 @@ import com.google.common.collect.ImmutableSet; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; +import jakarta.annotation.Nonnull; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; import org.graylog2.indexer.indexset.IndexSetConfig; import org.graylog2.indexer.indexset.IndexSetService; import org.graylog2.indexer.indexset.events.IndexSetCreatedEvent; import org.graylog2.indexer.indexset.events.IndexSetDeletedEvent; import org.graylog2.indexer.indices.TooManyAliasesException; -import jakarta.inject.Inject; -import jakarta.inject.Singleton; - import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -42,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Objects.requireNonNull; @@ -52,7 +53,7 @@ public class MongoIndexSetRegistry implements IndexSetRegistry { static class IndexSetsCache { private final IndexSetService indexSetService; - private AtomicReference>> indexSetConfigs; + private final AtomicReference>> indexSetConfigs; @Inject IndexSetsCache(IndexSetService indexSetService, @@ -143,12 +144,9 @@ public Set getForIndices(Collection indices) { @Override public Set getFromIndexConfig(Collection indexSetConfigs) { - final ImmutableSet.Builder mongoIndexSets = ImmutableSet.builder(); - for (IndexSetConfig config : indexSetConfigs) { - final MongoIndexSet mongoIndexSet = mongoIndexSetFactory.create(config); - mongoIndexSets.add(mongoIndexSet); - } - return ImmutableSet.copyOf(mongoIndexSets.build()); + return indexSetConfigs.stream() + .map(mongoIndexSetFactory::create) + .collect(Collectors.toUnmodifiableSet()); } @Override @@ -158,13 +156,9 @@ public IndexSet getDefault() { @Override public String[] getManagedIndices() { - final ImmutableSet.Builder indexNamesBuilder = ImmutableSet.builder(); - for (MongoIndexSet indexSet : findAllMongoIndexSets()) { - indexNamesBuilder.add(indexSet.getManagedIndices()); - } - - final ImmutableSet indexNames = indexNamesBuilder.build(); - return indexNames.toArray(new String[0]); + return findAllMongoIndexSets().stream() + .flatMap(indexSet -> Stream.of(indexSet.getManagedIndices())) + .toArray(String[]::new); } @Override @@ -180,38 +174,25 @@ public Map isManagedIndex(Collection indices) { } private boolean isManagedIndex(Collection indexSets, String index) { - for (IndexSet indexSet : indexSets) { - if (indexSet.isManagedIndex(index)) { - return true; - } - } - return false; + return indexSets.stream() + .anyMatch(indexSet -> indexSet.isManagedIndex(index)); + } + + private String[] doWithWritableIndices(Function fn) { + return findAllMongoIndexSets().stream() + .filter(indexSet -> indexSet.getConfig().isWritable()) + .map(fn) + .toArray(String[]::new); } @Override public String[] getIndexWildcards() { - final ImmutableSet.Builder wildcardsBuilder = ImmutableSet.builder(); - for (MongoIndexSet indexSet : findAllMongoIndexSets()) { - if (indexSet.getConfig().isWritable()) { - wildcardsBuilder.add(indexSet.getIndexWildcard()); - } - } - - final ImmutableSet wildcards = wildcardsBuilder.build(); - return wildcards.toArray(new String[0]); + return doWithWritableIndices(MongoIndexSet::getIndexWildcard); } @Override public String[] getWriteIndexAliases() { - final ImmutableSet.Builder indexNamesBuilder = ImmutableSet.builder(); - for (MongoIndexSet indexSet : findAllMongoIndexSets()) { - if (indexSet.getConfig().isWritable()) { - indexNamesBuilder.add(indexSet.getWriteIndexAlias()); - } - } - - final ImmutableSet indexNames = indexNamesBuilder.build(); - return indexNames.toArray(new String[0]); + return doWithWritableIndices(MongoIndexSet::getWriteIndexAlias); } @Override @@ -223,26 +204,18 @@ public boolean isUp() { @Override public boolean isCurrentWriteIndexAlias(String indexName) { - for (MongoIndexSet indexSet : findAllMongoIndexSets()) { - if (indexSet.isWriteIndexAlias(indexName)) { - return true; - } - } - - return false; + return findAllMongoIndexSets().stream() + .anyMatch(indexSet -> indexSet.isWriteIndexAlias(indexName)); } @Override public boolean isCurrentWriteIndex(String indexName) throws TooManyAliasesException { - for (MongoIndexSet indexSet : findAllMongoIndexSets()) { - if (indexSet.getActiveWriteIndex() != null && indexSet.getActiveWriteIndex().equals(indexName)) { - return true; - } - } - - return false; + return getForIndex(indexName) + .map(indexSet -> Objects.equals(indexSet.getActiveWriteIndex(), indexName)) + .orElse(false); } + @Nonnull @Override public Iterator iterator() { return getAll().iterator(); diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/MongoIndexSetRegistryTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/MongoIndexSetRegistryTest.java index 80349e8ea274..dbb842c943f6 100644 --- a/graylog2-server/src/test/java/org/graylog2/indexer/MongoIndexSetRegistryTest.java +++ b/graylog2-server/src/test/java/org/graylog2/indexer/MongoIndexSetRegistryTest.java @@ -16,6 +16,7 @@ */ package org.graylog2.indexer; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.eventbus.EventBus; import org.graylog2.indexer.indexset.IndexSetConfig; @@ -28,15 +29,22 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; +import java.time.ZonedDateTime; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; public class MongoIndexSetRegistryTest { @@ -266,4 +274,93 @@ public void isManagedIndexWithUnmanagedIndexReturnsFalse() { assertThat(indexSetRegistry.isManagedIndex("index")).isFalse(); } + + @Test + public void isCurrentWriteIndexSingleIndexSetConfigHappyPath() { + final String idxName = "index"; + final IndexSetConfig indexSetConfig = mock(IndexSetConfig.class); + final List indexSetConfigs = Collections.singletonList(indexSetConfig); + final MongoIndexSet indexSet = mock(MongoIndexSet.class); + when(mongoIndexSetFactory.create(indexSetConfig)).thenReturn(indexSet); + when(indexSetService.findAll()).thenReturn(indexSetConfigs); + when(indexSet.isManagedIndex(idxName)).thenReturn(true); + when(indexSet.getActiveWriteIndex()).thenReturn(idxName); + assertThat(indexSetRegistry.isCurrentWriteIndex(idxName)).isTrue(); + + verify(indexSet).isManagedIndex(idxName); + verify(indexSet).getActiveWriteIndex(); + verifyNoMoreInteractions(indexSetConfig, indexSet); + } + + @Test + public void isCurrentWriteIndexSingleIndexSetConfigNoneFound() { + final String idxName = "index"; + final IndexSetConfig indexSetConfig = mock(IndexSetConfig.class); + final List indexSetConfigs = Collections.singletonList(indexSetConfig); + final MongoIndexSet indexSet = mock(MongoIndexSet.class); + when(mongoIndexSetFactory.create(indexSetConfig)).thenReturn(indexSet); + when(indexSetService.findAll()).thenReturn(indexSetConfigs); + lenient().when(indexSet.getActiveWriteIndex()).thenReturn("non_existing_index"); + assertThat(indexSetRegistry.isCurrentWriteIndex(idxName)).isFalse(); + + verify(indexSet).isManagedIndex(idxName); + verify(indexSetService).findAll(); + verifyNoMoreInteractions(indexSetConfig, indexSet); + } + + @Test + public void isCurrentWriteIndexOnManyIndexSetConfigsNoneFoundDueToMismatchingIndexName() { + final String idxName = "index"; + final int noOfIndices = 2; + // The following constructs are necessary, because internally the IndexSetConfigs are handled as Set. + // Simply mocking a gazillion of them would de-duplicate, rendering the entire test useless. + final List indexSetConfigs = mkIndexSetConfigs(noOfIndices); + final ImmutableSet.Builder mockedMongosBuilder = ImmutableSet.builder(); + final AtomicReference matchingMongoMock = new AtomicReference<>(); + when(mongoIndexSetFactory.create(any(IndexSetConfig.class))) + .thenAnswer((Answer) invocation -> { + Object[] args = invocation.getArguments(); + final IndexSetConfig cfg = (IndexSetConfig) args[0]; + final MongoIndexSet mockedIndexSet = mock(MongoIndexSet.class); + lenient().when(mockedIndexSet.getConfig()).thenReturn(cfg); + final int currentIndex = Integer.parseInt(Objects.requireNonNull(cfg.id())); + if (currentIndex == noOfIndices - 1) { + when(mockedIndexSet.getActiveWriteIndex()).thenReturn(cfg.indexPrefix() + "_0"); + //Let the last MongoIndexSet be the one responsible to the index we're looking for: + when(mockedIndexSet.isManagedIndex(idxName)).thenReturn(true); + matchingMongoMock.getAndSet(mockedIndexSet); + } + mockedMongosBuilder.add(mockedIndexSet); + return mockedIndexSet; + } + ); + when(indexSetService.findAll()).thenReturn(indexSetConfigs); + assertThat(indexSetRegistry.isCurrentWriteIndex(idxName)).isFalse(); + + final ImmutableSet mockedMongos = mockedMongosBuilder.build(); + mockedMongos.forEach(mockedMongo -> verify(mockedMongo).isManagedIndex(idxName)); + verify(matchingMongoMock.get()).getActiveWriteIndex(); + verifyNoMoreInteractions(mockedMongos.toArray(new Object[0])); + } + + private List mkIndexSetConfigs(final int howMany) { + final ImmutableList.Builder configs = ImmutableList.builder(); + for (int i = 0; i < howMany; i++) { + configs.add(IndexSetConfig.builder() + .id(String.valueOf(i)) + .title("title " + i) + .indexPrefix("index" + i) + .shards(1) + .replicas(0) + .creationDate(ZonedDateTime.now()) + .indexOptimizationMaxNumSegments(1) + .indexOptimizationDisabled(false) + .indexAnalyzer("any") + .indexTemplateName("template" + i) + .build() + ); + + } + return configs.build(); + } } From 8e3941d8a038f392839799586570da568509f53f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:11:14 +0100 Subject: [PATCH 06/12] Bump io.netty:netty-bom from 4.1.115.Final to 4.1.116.Final (#21211) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.1.115.Final to 4.1.116.Final. - [Commits](https://github.com/netty/netty/compare/netty-4.1.115.Final...netty-4.1.116.Final) --- updated-dependencies: - dependency-name: io.netty:netty-bom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 320907e4f1c0..ab2d666c4f6f 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ 5.2.1 4.11.0 0.13 - 4.1.115.Final + 4.1.116.Final 2.0.69.Final 4.12.0 2.3 From 45e5c07e4caf62045f327ff2ee9dfb6b17762b56 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Wed, 18 Dec 2024 14:29:10 +0100 Subject: [PATCH 07/12] Better handling of intermediate CAs in datanode truststore (#21062) * TruststoreCreator adds the whole certchain, code cleanup * further truststore simplifications, unique aliases to prevent overriding already added certs * added changelog --- changelog/unreleased/pr-21062.toml | 4 + .../configuration/TruststoreCreator.java | 53 +++++++-- .../OpensearchSecurityConfigurationBean.java | 13 +-- .../configuration/TruststoreCreatorTest.java | 102 ++++++++++++++---- .../DatanodeSecurityTestUtils.java | 30 ++---- .../csr/FilesystemKeystoreInformation.java | 6 +- .../csr/InMemoryKeystoreInformation.java | 2 +- .../certutil/csr/KeystoreInformation.java | 4 +- 8 files changed, 152 insertions(+), 62 deletions(-) create mode 100644 changelog/unreleased/pr-21062.toml diff --git a/changelog/unreleased/pr-21062.toml b/changelog/unreleased/pr-21062.toml new file mode 100644 index 000000000000..0df01e48c0a3 --- /dev/null +++ b/changelog/unreleased/pr-21062.toml @@ -0,0 +1,4 @@ +type = "c" +message = "Better handling of intermediate CAs in datanode truststore" + +pulls = ["21062"] diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java b/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java index 57950d5dea11..5cbb5b1f07c3 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java @@ -22,6 +22,7 @@ import org.graylog.security.certutil.csr.InMemoryKeystoreInformation; import org.graylog.security.certutil.csr.KeystoreInformation; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Path; @@ -34,6 +35,7 @@ import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class TruststoreCreator { @@ -59,22 +61,33 @@ public static TruststoreCreator newEmpty() { } } - public TruststoreCreator addRootCert(final String name, KeystoreInformation keystoreInformation, - final String alias) throws GeneralSecurityException { - final X509Certificate rootCert; - try { - rootCert = findRootCert(keystoreInformation, alias); - } catch (Exception e) { - throw new RuntimeException(e); - } - this.truststore.setCertificateEntry(name, rootCert); - return this; + /** + * Originally we added only the root(=selfsigned) certificate to the truststore. But this causes problems with + * usage of intermediate CAs. There is nothing wrong adding the whole cert chain to the truststore. + * + * @param keystoreInformation access to the keystore, to obtain certificate chains by the given alias + * @param alias which certificate chain should we extract from the provided keystore + */ + public TruststoreCreator addFromKeystore(KeystoreInformation keystoreInformation, + final String alias) throws IOException, GeneralSecurityException { + final KeyStore keystore = keystoreInformation.loadKeystore(); + final Certificate[] chain = keystore.getCertificateChain(alias); + final List x509Certs = toX509Certs(chain); + return addCertificates(x509Certs); + } + + @Nonnull + private static List toX509Certs(Certificate[] certs) { + return Arrays.stream(certs) + .filter(c -> c instanceof X509Certificate) + .map(c -> (X509Certificate) c) + .toList(); } public TruststoreCreator addCertificates(List trustedCertificates) { trustedCertificates.forEach(cert -> { try { - this.truststore.setCertificateEntry(cert.getSubjectX500Principal().getName(), cert); + this.truststore.setCertificateEntry(generateAlias(this.truststore, cert), cert); } catch (KeyStoreException e) { throw new RuntimeException(e); } @@ -82,6 +95,24 @@ public TruststoreCreator addCertificates(List trustedCertificat return this; } + /** + * Alias has no meaning for the trust and validation purposes in the truststore. It's there only for managing + * the truststore content. We just need to make sure that we are using unique aliases, otherwise the + * truststore would override already present certificates. + * + * If there is no collision, we use the cname as given in the cert. In case of collisions, we'll append _i, + * where is index an incremented till it's unique in the truststore. + */ + private static String generateAlias(KeyStore truststore, X509Certificate cert) throws KeyStoreException { + AtomicInteger counter = new AtomicInteger(1); + final String cname = cert.getSubjectX500Principal().getName(); + String alias = cname; + while (truststore.containsAlias(alias)) { + alias = cname + "_" + counter.getAndIncrement(); + } + return alias; + } + public FilesystemKeystoreInformation persist(final Path truststorePath, final char[] truststorePassword) throws IOException, GeneralSecurityException { try (final FileOutputStream fileOutputStream = new FileOutputStream(truststorePath.toFile())) { diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java index f026d2bda49f..b48bbddd446e 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java @@ -36,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; import java.security.GeneralSecurityException; import java.security.KeyStore; @@ -112,10 +113,10 @@ public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationP try { configurationBuilder.httpCertificate(cert); configurationBuilder.withConfigFile(new KeystoreConfigFile(Path.of(TARGET_DATANODE_HTTP_KEYSTORE_FILENAME), cert)); - truststoreCreator.addRootCert("http-cert", cert, CertConstants.DATANODE_KEY_ALIAS); + truststoreCreator.addFromKeystore(cert, CertConstants.DATANODE_KEY_ALIAS); logCertificateInformation("HTTP certificate", cert); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); + } catch (GeneralSecurityException | IOException e) { + throw new OpensearchConfigurationException(e); } }); @@ -123,10 +124,10 @@ public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationP try { configurationBuilder.transportCertificate(cert); configurationBuilder.withConfigFile(new KeystoreConfigFile(Path.of(TARGET_DATANODE_TRANSPORT_KEYSTORE_FILENAME), cert)); - truststoreCreator.addRootCert("transport-cert", cert, CertConstants.DATANODE_KEY_ALIAS); + truststoreCreator.addFromKeystore(cert, CertConstants.DATANODE_KEY_ALIAS); logCertificateInformation("Transport certificate", cert); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); + } catch (GeneralSecurityException | IOException e) { + throw new OpensearchConfigurationException(e); } }); diff --git a/data-node/src/test/java/org/graylog/datanode/configuration/TruststoreCreatorTest.java b/data-node/src/test/java/org/graylog/datanode/configuration/TruststoreCreatorTest.java index 4520a7a0602e..715b7410b596 100644 --- a/data-node/src/test/java/org/graylog/datanode/configuration/TruststoreCreatorTest.java +++ b/data-node/src/test/java/org/graylog/datanode/configuration/TruststoreCreatorTest.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import jakarta.annotation.Nonnull; import org.apache.commons.lang3.RandomStringUtils; import org.assertj.core.api.Assertions; import org.bouncycastle.asn1.x500.X500Name; @@ -28,12 +29,19 @@ import org.bouncycastle.operator.OperatorCreationException; import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; import org.graylog.security.certutil.CertConstants; +import org.graylog.security.certutil.CertRequest; +import org.graylog.security.certutil.CertificateGenerator; +import org.graylog.security.certutil.KeyPair; import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; +import org.graylog.security.certutil.csr.InMemoryKeystoreInformation; +import org.graylog.security.certutil.csr.KeystoreInformation; import org.graylog.security.certutil.keystore.storage.KeystoreFileStorage; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import java.io.FileOutputStream; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.math.BigInteger; import java.nio.file.Path; @@ -41,11 +49,15 @@ import java.security.KeyPairGenerator; import java.security.KeyStore; import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; import java.security.cert.Certificate; +import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Optional; @@ -61,12 +73,12 @@ public class TruststoreCreatorTest { @Test void testTrustStoreCreation(@TempDir Path tempDir) throws Exception { - final FilesystemKeystoreInformation root = createKeystore(tempDir.resolve("root.p12"), "root", "CN=ROOT", BigInteger.ONE); - final FilesystemKeystoreInformation boot = createKeystore(tempDir.resolve("boot.p12"), "boot", "CN=BOOT", BigInteger.TWO); + final KeystoreInformation root = createKeystore(tempDir.resolve("root.p12"), "root", "CN=ROOT", BigInteger.ONE); + final KeystoreInformation boot = createKeystore(tempDir.resolve("boot.p12"), "boot", "CN=BOOT", BigInteger.TWO); final FilesystemKeystoreInformation truststore = TruststoreCreator.newEmpty() - .addRootCert("root", root, "root") - .addRootCert("boot", boot, "boot") + .addFromKeystore(root, "root") + .addFromKeystore(boot, "boot") .persist(tempDir.resolve("truststore.sec"), "caramba! caramba!".toCharArray()); @@ -79,11 +91,11 @@ void testTrustStoreCreation(@TempDir Path tempDir) throws Exception { final KeyStore keyStore = keyStoreOptional.get(); assertThat(ImmutableList.copyOf(keyStore.aliases().asIterator())) - .containsOnly("root", "boot"); + .containsOnly("cn=root", "cn=boot"); - final Certificate rootCert = keyStore.getCertificate("root"); + final Certificate rootCert = keyStore.getCertificate("cn=root"); verifyCertificate(rootCert, "CN=ROOT", BigInteger.ONE); - final Certificate bootCert = keyStore.getCertificate("boot"); + final Certificate bootCert = keyStore.getCertificate("cn=boot"); verifyCertificate(bootCert, "CN=BOOT", BigInteger.TWO); } @@ -98,7 +110,7 @@ void testDefaultJvm() throws KeyStoreException { @Test void testAdditionalCertificates(@TempDir Path tempDir) throws GeneralSecurityException, IOException, OperatorCreationException { - final FilesystemKeystoreInformation root = createKeystore(tempDir.resolve("root.p12"), "something-unknown", "CN=ROOT", BigInteger.ONE); + final KeystoreInformation root = createKeystore(tempDir.resolve("root.p12"), "something-unknown", "CN=ROOT", BigInteger.ONE); final X509Certificate cert = (X509Certificate) root.loadKeystore().getCertificate("something-unknown"); final FilesystemKeystoreInformation truststore = TruststoreCreator.newEmpty() @@ -111,7 +123,64 @@ void testAdditionalCertificates(@TempDir Path tempDir) throws GeneralSecurityExc Assertions.assertThat(alias) .isNotNull() .isEqualTo("cn=root"); + } + + @Test + void testIntermediateCa() throws Exception { + final KeyPair ca = CertificateGenerator.generate(CertRequest.selfSigned("my-ca").isCA(true).validity(Duration.ofDays(100))); + final KeyPair intermediateCa = CertificateGenerator.generate(CertRequest.signed("intermediate", ca).isCA(true).validity(Duration.ofDays(100))); + final KeyPair nodeKeys = CertificateGenerator.generate(CertRequest.signed("my-node", intermediateCa).isCA(false).validity(Duration.ofDays(100))); + + + final InMemoryKeystoreInformation keystoreInformation = createInMemoryKeystore(nodeKeys, intermediateCa); + final KeyStore truststore = TruststoreCreator.newEmpty() + .addFromKeystore(keystoreInformation, "my-node") + .getTruststore(); + + final X509TrustManager defaultTrustManager = createTrustManager(truststore); + + Assertions.assertThatNoException().isThrownBy(() -> defaultTrustManager.checkServerTrusted(new X509Certificate[]{nodeKeys.certificate()}, "RSA")); + + final KeyPair fakeNodeKeys = CertificateGenerator.generate(CertRequest.selfSigned("my-fake-node").isCA(false).validity(Duration.ofDays(100))); + Assertions.assertThatThrownBy(() -> defaultTrustManager.checkServerTrusted(new X509Certificate[]{fakeNodeKeys.certificate()}, "RSA")) + .isInstanceOf(CertificateException.class); + } + + @Test + void testDuplicateCname() throws Exception { + final KeyPair ca1 = CertificateGenerator.generate(CertRequest.selfSigned("my-ca").isCA(true).validity(Duration.ofDays(90))); + final KeyPair ca2 = CertificateGenerator.generate(CertRequest.selfSigned("my-ca").isCA(true).validity(Duration.ofDays(90))); + final KeyPair ca3 = CertificateGenerator.generate(CertRequest.selfSigned("my-ca").isCA(true).validity(Duration.ofDays(90))); + + final KeyStore truststore = TruststoreCreator.newEmpty() + .addCertificates(List.of(ca1.certificate())) + .addCertificates(List.of(ca2.certificate())) + .addCertificates(List.of(ca3.certificate())) + .getTruststore(); + + Assertions.assertThat(Collections.list(truststore.aliases())) + .hasSize(3) + .contains("cn=my-ca") + .contains("cn=my-ca_1") + .contains("cn=my-ca_2"); + } + + private static X509TrustManager createTrustManager(KeyStore caTruststore) throws NoSuchAlgorithmException, KeyStoreException { + final TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(caTruststore); + final TrustManager[] trustManagers = tmf.getTrustManagers(); + return (X509TrustManager) trustManagers[0]; + } + + @SuppressWarnings("deprecation") + @Nonnull + private static InMemoryKeystoreInformation createInMemoryKeystore(KeyPair nodeKeys, KeyPair intermediate) throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException { + final char[] password = RandomStringUtils.randomAlphabetic(256).toCharArray(); + KeyStore keystore = KeyStore.getInstance(CertConstants.PKCS12); + keystore.load(null, null); + keystore.setKeyEntry("my-node", nodeKeys.privateKey(), password, new Certificate[]{nodeKeys.certificate(), intermediate.certificate()}); + return new InMemoryKeystoreInformation(keystore, password); } private void verifyCertificate(final Certificate rootCert, final String cnName, final BigInteger serialNumber) { @@ -124,7 +193,8 @@ private void verifyCertificate(final Certificate rootCert, final String cnName, assertEquals(cnName, x509Certificate.getIssuerX500Principal().getName()); } - private FilesystemKeystoreInformation createKeystore(Path path, String alias, final String cnName, final BigInteger serialNumber) throws GeneralSecurityException, OperatorCreationException, IOException { + @SuppressWarnings("deprecation") + private KeystoreInformation createKeystore(Path path, String alias, final String cnName, final BigInteger serialNumber) throws GeneralSecurityException, OperatorCreationException, IOException { KeyPairGenerator keyGen = KeyPairGenerator.getInstance(KEY_GENERATION_ALGORITHM); java.security.KeyPair certKeyPair = keyGen.generateKeyPair(); X500Name name = new X500Name(cnName); @@ -142,17 +212,13 @@ private FilesystemKeystoreInformation createKeystore(Path path, String alias, fi final X509Certificate signedCert = new JcaX509CertificateConverter().getCertificate(certHolder); - KeyStore trustStore = KeyStore.getInstance(CertConstants.PKCS12); - trustStore.load(null, null); + KeyStore keyStore = KeyStore.getInstance(CertConstants.PKCS12); + keyStore.load(null, null); final char[] password = RandomStringUtils.randomAlphabetic(256).toCharArray(); - trustStore.setKeyEntry(alias, certKeyPair.getPrivate(), password, new Certificate[]{signedCert}); - + keyStore.setKeyEntry(alias, certKeyPair.getPrivate(), password, new Certificate[]{signedCert}); - try (final FileOutputStream fileOutputStream = new FileOutputStream(path.toFile())) { - trustStore.store(fileOutputStream, password); - } - return new FilesystemKeystoreInformation(path, password); + return new InMemoryKeystoreInformation(keyStore, password); } } diff --git a/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSecurityTestUtils.java b/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSecurityTestUtils.java index 8bd3bf09ce6f..b4237d5a60ad 100644 --- a/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSecurityTestUtils.java +++ b/data-node/src/test/java/org/graylog/datanode/integration/DatanodeSecurityTestUtils.java @@ -17,41 +17,29 @@ package org.graylog.datanode.integration; import org.apache.commons.lang3.RandomStringUtils; +import org.graylog.datanode.configuration.TruststoreCreator; import org.graylog.security.certutil.CertutilCa; import org.graylog.security.certutil.CertutilCert; import org.graylog.security.certutil.CertutilHttp; import org.graylog.security.certutil.console.TestableConsole; import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; +import org.graylog.security.certutil.csr.KeystoreInformation; -import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Path; import java.security.GeneralSecurityException; import java.security.KeyStore; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; import java.util.Enumeration; public class DatanodeSecurityTestUtils { - public static KeyStore buildTruststore(FilesystemKeystoreInformation ca) throws IOException, GeneralSecurityException { - try (FileInputStream fis = new FileInputStream(ca.location().toFile())) { - - KeyStore caKeystore = KeyStore.getInstance("PKCS12"); - caKeystore.load(fis, ca.password()); - - KeyStore trustStore = KeyStore.getInstance("PKCS12"); - trustStore.load(null, null); - - final Enumeration aliases = caKeystore.aliases(); - while (aliases.hasMoreElements()) { - final String alias = aliases.nextElement(); - final Certificate cert = caKeystore.getCertificate(alias); - if (cert instanceof final X509Certificate x509Certificate) { - trustStore.setCertificateEntry(alias, x509Certificate); - } - } - return trustStore; + public static KeyStore buildTruststore(KeystoreInformation ca) throws IOException, GeneralSecurityException { + final TruststoreCreator truststoreCreator = TruststoreCreator.newEmpty(); + final Enumeration aliases = ca.loadKeystore().aliases(); + while (aliases.hasMoreElements()) { + final String alias = aliases.nextElement(); + truststoreCreator.addFromKeystore(ca, alias); } + return truststoreCreator.getTruststore(); } public static FilesystemKeystoreInformation generateCa(Path dir) { diff --git a/graylog2-server/src/main/java/org/graylog/security/certutil/csr/FilesystemKeystoreInformation.java b/graylog2-server/src/main/java/org/graylog/security/certutil/csr/FilesystemKeystoreInformation.java index f46d7127bb5f..2fe48db6beff 100644 --- a/graylog2-server/src/main/java/org/graylog/security/certutil/csr/FilesystemKeystoreInformation.java +++ b/graylog2-server/src/main/java/org/graylog/security/certutil/csr/FilesystemKeystoreInformation.java @@ -19,10 +19,8 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Path; +import java.security.GeneralSecurityException; import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; import java.util.Arrays; import java.util.Objects; @@ -38,7 +36,7 @@ public FilesystemKeystoreInformation(Path location, char[] password) { } @Override - public KeyStore loadKeystore() throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException { + public KeyStore loadKeystore() throws IOException, GeneralSecurityException { KeyStore keyStore = KeyStore.getInstance(PKCS12); try (FileInputStream fis = new FileInputStream(location.toFile())) { keyStore.load(fis, password); diff --git a/graylog2-server/src/main/java/org/graylog/security/certutil/csr/InMemoryKeystoreInformation.java b/graylog2-server/src/main/java/org/graylog/security/certutil/csr/InMemoryKeystoreInformation.java index f4f216ddaf3b..dad8af8b0a9d 100644 --- a/graylog2-server/src/main/java/org/graylog/security/certutil/csr/InMemoryKeystoreInformation.java +++ b/graylog2-server/src/main/java/org/graylog/security/certutil/csr/InMemoryKeystoreInformation.java @@ -29,7 +29,7 @@ public InMemoryKeystoreInformation(KeyStore keyStore, char[] password) { } @Override - public KeyStore loadKeystore() throws Exception { + public KeyStore loadKeystore() { return keyStore; } diff --git a/graylog2-server/src/main/java/org/graylog/security/certutil/csr/KeystoreInformation.java b/graylog2-server/src/main/java/org/graylog/security/certutil/csr/KeystoreInformation.java index 746eb4ef773d..424d94309e30 100644 --- a/graylog2-server/src/main/java/org/graylog/security/certutil/csr/KeystoreInformation.java +++ b/graylog2-server/src/main/java/org/graylog/security/certutil/csr/KeystoreInformation.java @@ -16,11 +16,13 @@ */ package org.graylog.security.certutil.csr; +import java.io.IOException; +import java.security.GeneralSecurityException; import java.security.KeyStore; public interface KeystoreInformation { - KeyStore loadKeystore() throws Exception; + KeyStore loadKeystore() throws IOException, GeneralSecurityException; char[] password(); } From 40914226e58b202c3bd70c40fb80c402ad980d83 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Thu, 19 Dec 2024 08:22:57 +0100 Subject: [PATCH 08/12] Handle path prefix when serving web interface assets. (#21104) * Handle path prefix when serving web interface assets. * Adding tests for frontend asset consistency. * Adding changelog snippet. --- changelog/unreleased/issue-21015.toml | 5 ++ .../WebInterfaceAssetsResourceBase.java | 59 +++++++++++++++++++ .../WebInterfaceAssetsResourceIT.java | 36 +++++++++++ ...ebInterfaceAssetsResourceWithPrefixIT.java | 39 ++++++++++++ .../resources/WebInterfaceAssetsResource.java | 58 ++++++++++++------ .../MavenProjectDirProviderWithFrontend.java | 24 ++++++++ .../graylognode/NodeContainerFactory.java | 15 +++-- 7 files changed, 213 insertions(+), 23 deletions(-) create mode 100644 changelog/unreleased/issue-21015.toml create mode 100644 full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceBase.java create mode 100644 full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceIT.java create mode 100644 full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceWithPrefixIT.java create mode 100644 graylog2-server/src/test/java/org/graylog/testing/completebackend/MavenProjectDirProviderWithFrontend.java diff --git a/changelog/unreleased/issue-21015.toml b/changelog/unreleased/issue-21015.toml new file mode 100644 index 000000000000..56a2e06c0c57 --- /dev/null +++ b/changelog/unreleased/issue-21015.toml @@ -0,0 +1,5 @@ +type = "f" +message = "Handle path prefix when serving web interface assets." + +issues = ["21015"] +pulls = ["21104"] diff --git a/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceBase.java b/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceBase.java new file mode 100644 index 000000000000..df481f380b89 --- /dev/null +++ b/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceBase.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.web.resources; + +import io.restassured.http.ContentType; +import io.restassured.specification.RequestSpecification; +import org.apache.http.HttpStatus; +import org.graylog.testing.completebackend.apis.GraylogApis; + +import static io.restassured.RestAssured.given; + +public abstract class WebInterfaceAssetsResourceBase { + private final GraylogApis apis; + + protected WebInterfaceAssetsResourceBase(GraylogApis apis) { + this.apis = apis; + } + + private RequestSpecification backend() { + return given() + .baseUri(apis.backend().uri()) + .port(apis.backend().apiPort()); + } + + protected void testFrontend(String prefix) { + final var scriptSrcs = backend() + .get(prefix) + .then() + .assertThat() + .statusCode(HttpStatus.SC_OK) + .contentType(ContentType.HTML) + .extract() + .htmlPath() + .getList("html.body.script*.@src"); + + scriptSrcs.forEach(src -> { + backend() + .get(src) + .then() + .assertThat() + .statusCode(HttpStatus.SC_OK) + .contentType(ContentType.JSON); + }); + } +} diff --git a/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceIT.java b/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceIT.java new file mode 100644 index 000000000000..f0712fdee5a9 --- /dev/null +++ b/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceIT.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.web.resources; + +import org.graylog.testing.completebackend.MavenProjectDirProviderWithFrontend; +import org.graylog.testing.completebackend.apis.GraylogApis; +import org.graylog.testing.containermatrix.SearchServer; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration; + +@ContainerMatrixTestsConfiguration(mavenProjectDirProvider = MavenProjectDirProviderWithFrontend.class, + searchVersions = {SearchServer.DATANODE_DEV}) +public class WebInterfaceAssetsResourceIT extends WebInterfaceAssetsResourceBase { + public WebInterfaceAssetsResourceIT(GraylogApis graylogApis) { + super(graylogApis); + } + + @ContainerMatrixTest + void testIndexHtml() { + testFrontend("/"); + } +} diff --git a/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceWithPrefixIT.java b/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceWithPrefixIT.java new file mode 100644 index 000000000000..41fb247cc148 --- /dev/null +++ b/full-backend-tests/src/test/java/org/graylog2/web/resources/WebInterfaceAssetsResourceWithPrefixIT.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.web.resources; + +import org.graylog.testing.completebackend.MavenProjectDirProviderWithFrontend; +import org.graylog.testing.completebackend.apis.GraylogApis; +import org.graylog.testing.containermatrix.SearchServer; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration; + +@ContainerMatrixTestsConfiguration(mavenProjectDirProvider = MavenProjectDirProviderWithFrontend.class, + searchVersions = {SearchServer.DATANODE_DEV}, + additionalConfigurationParameters = { + @ContainerMatrixTestsConfiguration.ConfigurationParameter(key = "GRAYLOG_HTTP_PUBLISH_URI", value = "http://localhost:9000/graylog") + }) +public class WebInterfaceAssetsResourceWithPrefixIT extends WebInterfaceAssetsResourceBase { + public WebInterfaceAssetsResourceWithPrefixIT(GraylogApis graylogApis) { + super(graylogApis); + } + + @ContainerMatrixTest + void testIndexHtml() { + testFrontend("/graylog/"); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/web/resources/WebInterfaceAssetsResource.java b/graylog2-server/src/main/java/org/graylog2/web/resources/WebInterfaceAssetsResource.java index 82274bd18a18..31e5a6439062 100644 --- a/graylog2-server/src/main/java/org/graylog2/web/resources/WebInterfaceAssetsResource.java +++ b/graylog2-server/src/main/java/org/graylog2/web/resources/WebInterfaceAssetsResource.java @@ -22,19 +22,8 @@ import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; import com.google.common.io.Resources; -import org.glassfish.jersey.server.ContainerRequest; -import org.graylog2.plugin.Plugin; -import org.graylog2.shared.rest.resources.csp.CSP; -import org.graylog2.shared.rest.resources.csp.CSPDynamicFeature; -import org.graylog2.web.IndexHtmlGenerator; -import org.graylog2.web.PluginAssets; - -import javax.activation.MimetypesFileTypeMap; -import javax.annotation.Nonnull; - import jakarta.inject.Inject; import jakarta.inject.Singleton; - import jakarta.ws.rs.GET; import jakarta.ws.rs.NotFoundException; import jakarta.ws.rs.Path; @@ -46,7 +35,17 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Request; import jakarta.ws.rs.core.Response; +import org.glassfish.jersey.server.ContainerRequest; +import org.graylog2.configuration.HttpConfiguration; +import org.graylog2.plugin.Plugin; +import org.graylog2.rest.RestTools; +import org.graylog2.shared.rest.resources.csp.CSP; +import org.graylog2.shared.rest.resources.csp.CSPDynamicFeature; +import org.graylog2.web.IndexHtmlGenerator; +import org.graylog2.web.PluginAssets; +import javax.activation.MimetypesFileTypeMap; +import javax.annotation.Nonnull; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -73,15 +72,20 @@ @CSP(group = CSP.DEFAULT) public class WebInterfaceAssetsResource { private final MimetypesFileTypeMap mimeTypes; + private final HttpConfiguration httpConfiguration; private final IndexHtmlGenerator indexHtmlGenerator; private final Set plugins; private final LoadingCache fileSystemCache; @Inject - public WebInterfaceAssetsResource(IndexHtmlGenerator indexHtmlGenerator, Set plugins, MimetypesFileTypeMap mimeTypes) { + public WebInterfaceAssetsResource(IndexHtmlGenerator indexHtmlGenerator, + Set plugins, + MimetypesFileTypeMap mimeTypes, + HttpConfiguration httpConfiguration) { this.indexHtmlGenerator = indexHtmlGenerator; this.plugins = plugins; this.mimeTypes = requireNonNull(mimeTypes); + this.httpConfiguration = httpConfiguration; this.fileSystemCache = CacheBuilder.newBuilder() .maximumSize(1024) .build(new CacheLoader<>() { @@ -103,16 +107,18 @@ public FileSystem load(@Nonnull URI key) throws Exception { @Path("assets/plugin/{plugin}/{filename}") @GET public Response get(@Context Request request, + @Context HttpHeaders headers, @PathParam("plugin") String pluginName, @PathParam("filename") String filename) { final Plugin plugin = getPluginForName(pluginName) .orElseThrow(() -> new NotFoundException("Couldn't find plugin " + pluginName)); + final var filenameWithoutSuffix = trimBasePath(filename, headers); try { - final URL resourceUrl = getResourceUri(true, filename, plugin.metadata().getClass()); - return getResponse(request, filename, resourceUrl, true); + final URL resourceUrl = getResourceUri(true, filenameWithoutSuffix, plugin.metadata().getClass()); + return getResponse(request, filenameWithoutSuffix, resourceUrl, true); } catch (URISyntaxException | IOException e) { - throw new NotFoundException("Couldn't find " + filename + " in plugin " + pluginName, e); + throw new NotFoundException("Couldn't find " + filenameWithoutSuffix + " in plugin " + pluginName, e); } } @@ -125,17 +131,33 @@ private Optional getPluginForName(String pluginName) { public Response get(@Context ContainerRequest request, @Context HttpHeaders headers, @PathParam("filename") String filename) { + final var filenameWithoutSuffix = trimBasePath(filename, headers); try { - final URL resourceUrl = getResourceUri(false, filename, this.getClass()); - return getResponse(request, filename, resourceUrl, false); + final URL resourceUrl = getResourceUri(false, filenameWithoutSuffix, this.getClass()); + return getResponse(request, filenameWithoutSuffix, resourceUrl, false); } catch (IOException | URISyntaxException e) { return generateIndexHtml(headers, (String) request.getProperty(CSPDynamicFeature.CSP_NONCE_PROPERTY)); } + + } + + private String trimBasePath(String filename, HttpHeaders headers) { + final String baseUriPath = removeTrailingSlash(RestTools.buildRelativeExternalUri(headers.getRequestHeaders(), httpConfiguration.getHttpExternalUri()).getPath()); + return filename.startsWith(baseUriPath) ? filename.substring(baseUriPath.length()) : filename; + } + + private String removeTrailingSlash(String basePath) { + if (basePath == null || !basePath.endsWith("/")) { + return basePath; + } + + return basePath.substring(0, basePath.length() - 1); } @GET @Path("{filename:.*}") - public Response getIndex(@Context ContainerRequest request, @Context HttpHeaders headers) { + public Response getIndex(@Context ContainerRequest request, + @Context HttpHeaders headers) { final URI originalLocation = request.getRequestUri(); return get(request, headers, originalLocation.getPath()); } diff --git a/graylog2-server/src/test/java/org/graylog/testing/completebackend/MavenProjectDirProviderWithFrontend.java b/graylog2-server/src/test/java/org/graylog/testing/completebackend/MavenProjectDirProviderWithFrontend.java new file mode 100644 index 000000000000..84ee7c9411e9 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/testing/completebackend/MavenProjectDirProviderWithFrontend.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.testing.completebackend; + +public class MavenProjectDirProviderWithFrontend extends DefaultMavenProjectDirProvider { + @Override + public boolean includeFrontend() { + return true; + } +} diff --git a/graylog2-server/src/test/java/org/graylog/testing/graylognode/NodeContainerFactory.java b/graylog2-server/src/test/java/org/graylog/testing/graylognode/NodeContainerFactory.java index 1277517c9800..593c143d260d 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/graylognode/NodeContainerFactory.java +++ b/graylog2-server/src/test/java/org/graylog/testing/graylognode/NodeContainerFactory.java @@ -26,10 +26,10 @@ import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import org.testcontainers.containers.wait.strategy.WaitAllStrategy; import org.testcontainers.images.builder.ImageFromDockerfile; -import org.testcontainers.utility.MountableFile; import java.io.File; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -38,6 +38,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.stream.StreamSupport; import static java.time.temporal.ChronoUnit.SECONDS; @@ -165,7 +166,7 @@ private static GenericContainer createRunningContainer(NodeContainerConfig co config.mavenProjectDirProvider.getFilesToAddToBinDir().forEach(filename -> { final Path originalPath = fileCopyBaseDir.resolve(filename); final String containerPath = GRAYLOG_HOME + "/bin/" + originalPath.getFileName(); - container.addFileSystemBind(originalPath.toString(), containerPath.toString(), BindMode.READ_ONLY); + container.addFileSystemBind(originalPath.toString(), containerPath, BindMode.READ_ONLY); }); addEnabledFeatureFlagsToContainerEnv(config, container); @@ -193,7 +194,11 @@ private static WaitAllStrategy getWaitStrategy(Map env) { if(indexerIsPredefined(env)) { // we have defined an indexer, no preflight will occur, let's wait for the full boot with index ranges // To be able to search for data we need the index ranges to be computed. Since this is an async // background job, we need to wait until they have been created. - waitAllStrategy.withStrategy(waitForIndexRangesStrategy()); + final var baseUrl = Optional.ofNullable(env.get("GRAYLOG_HTTP_PUBLISH_URI")) + .map(URI::create) + .map(URI::getPath) + .orElse(""); + waitAllStrategy.withStrategy(waitForIndexRangesStrategy(baseUrl)); } return waitAllStrategy; @@ -203,10 +208,10 @@ private static boolean indexerIsPredefined(Map env) { return !env.getOrDefault(ENV_GRAYLOG_ELASTICSEARCH_HOSTS, "").isBlank(); } - private static HttpWaitStrategy waitForIndexRangesStrategy() { + private static HttpWaitStrategy waitForIndexRangesStrategy(String urlPrefix) { return new HttpWaitStrategy() .forPort(API_PORT) - .forPath("/api/system/indices/ranges") + .forPath(urlPrefix + "/api/system/indices/ranges") .withMethod("GET") .withBasicCredentials("admin", "admin") .forResponsePredicate(body -> { From 41fb93cc537222f9f4fd91da5981ed73918b428c Mon Sep 17 00:00:00 2001 From: Linus Pahl <46300478+linuspahl@users.noreply.github.com> Date: Thu, 19 Dec 2024 08:33:52 +0100 Subject: [PATCH 09/12] Implement `DefaultQueryParamProvider`. (#21213) * Implement `DefaultQueryParamProvider`. * Fixing import path --- .../hooks/useUrlQueryFilters.test.tsx | 8 +++--- .../ProfilesList.test.tsx | 7 ++--- .../IndexSetFieldTypesList.test.tsx | 8 +++--- .../SetProfileModal.test.tsx | 7 ++--- .../StreamsOverview/StreamsOverview.test.tsx | 7 ++--- .../src/pages/IndexSetFieldTypesPage.test.tsx | 8 +++--- graylog2-web-interface/src/routing/App.tsx | 7 ++--- .../src/routing/DefaultQueryParamProvider.tsx | 28 +++++++++++++++++++ .../DashboardsOverview.test.tsx | 8 +++--- .../src/views/pages/DashboardsPage.test.tsx | 8 +++--- 10 files changed, 60 insertions(+), 36 deletions(-) create mode 100644 graylog2-web-interface/src/routing/DefaultQueryParamProvider.tsx diff --git a/graylog2-web-interface/src/components/common/EntityFilters/hooks/useUrlQueryFilters.test.tsx b/graylog2-web-interface/src/components/common/EntityFilters/hooks/useUrlQueryFilters.test.tsx index 1b50c78f1cea..3681ff8bbe58 100644 --- a/graylog2-web-interface/src/components/common/EntityFilters/hooks/useUrlQueryFilters.test.tsx +++ b/graylog2-web-interface/src/components/common/EntityFilters/hooks/useUrlQueryFilters.test.tsx @@ -17,10 +17,10 @@ import { renderHook } from 'wrappedTestingLibrary/hooks'; import { OrderedMap } from 'immutable'; import * as React from 'react'; -import { useQueryParam, QueryParamProvider } from 'use-query-params'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; +import { useQueryParam } from 'use-query-params'; import { MemoryRouter } from 'react-router-dom'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; import { asMock } from 'helpers/mocking'; import useUrlQueryFilters from './useUrlQueryFilters'; @@ -33,9 +33,9 @@ jest.mock('use-query-params', () => ({ describe('useUrlQueryFilters', () => { const wrapper = ({ children }: { children: React.ReactNode }) => ( - + {children} - + ); diff --git a/graylog2-web-interface/src/components/indices/IndexSetFieldTypeProfiles/ProfilesList.test.tsx b/graylog2-web-interface/src/components/indices/IndexSetFieldTypeProfiles/ProfilesList.test.tsx index 52ee05e555ec..55fce130a735 100644 --- a/graylog2-web-interface/src/components/indices/IndexSetFieldTypeProfiles/ProfilesList.test.tsx +++ b/graylog2-web-interface/src/components/indices/IndexSetFieldTypeProfiles/ProfilesList.test.tsx @@ -16,8 +16,6 @@ */ import * as React from 'react'; import { render, screen, fireEvent, within } from 'wrappedTestingLibrary'; -import { QueryParamProvider } from 'use-query-params'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; import asMock from 'helpers/mocking/AsMock'; import useUserLayoutPreferences from 'components/common/EntityDataTable/hooks/useUserLayoutPreferences'; @@ -28,6 +26,7 @@ import useFieldTypesForMappings from 'views/logic/fieldactions/ChangeFieldType/h import { profile1, attributes, profile2 } from 'fixtures/indexSetFieldTypeProfiles'; import ProfilesList from 'components/indices/IndexSetFieldTypeProfiles/ProfilesList'; import useFetchEntities from 'components/common/PaginatedEntityTable/useFetchEntities'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; const getData = (list = [profile1]) => ( { @@ -40,11 +39,11 @@ const getData = (list = [profile1]) => ( ); const renderIndexSetFieldTypeProfilesList = () => render( - + , - , + , ); jest.mock('routing/useParams', () => jest.fn()); diff --git a/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/IndexSetFieldTypesList.test.tsx b/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/IndexSetFieldTypesList.test.tsx index f61825c10790..d7bf5a9d5157 100644 --- a/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/IndexSetFieldTypesList.test.tsx +++ b/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/IndexSetFieldTypesList.test.tsx @@ -16,8 +16,7 @@ */ import * as React from 'react'; import { render, screen, fireEvent, within } from 'wrappedTestingLibrary'; -import { useQueryParam, QueryParamProvider } from 'use-query-params'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; +import { useQueryParam } from 'use-query-params'; import { MockStore } from 'helpers/mocking'; import useParams from 'routing/useParams'; @@ -40,6 +39,7 @@ import useProfile from 'components/indices/IndexSetFieldTypeProfiles/hooks/usePr import useIndexProfileWithMappingsByField from 'components/indices/IndexSetFieldTypes/hooks/useIndexProfileWithMappingsByField'; import useProfileOptions from 'components/indices/IndexSetFieldTypeProfiles/hooks/useProfileOptions'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; const getData = (list = [defaultField]) => ( { @@ -52,11 +52,11 @@ const getData = (list = [defaultField]) => ( ); const renderIndexSetFieldTypesList = () => render( - + , - , + , ); jest.mock('stores/indices/IndexSetsStore', () => ({ diff --git a/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/SetProfileModal.test.tsx b/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/SetProfileModal.test.tsx index 55e9de9394c9..06ec2062d318 100644 --- a/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/SetProfileModal.test.tsx +++ b/graylog2-web-interface/src/components/indices/IndexSetFieldTypes/SetProfileModal.test.tsx @@ -16,8 +16,6 @@ */ import * as React from 'react'; import { render, screen, fireEvent } from 'wrappedTestingLibrary'; -import { QueryParamProvider } from 'use-query-params'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; import selectEvent from 'react-select-event'; import useSetIndexSetProfileMutation from 'components/indices/IndexSetFieldTypes/hooks/useSetIndexSetProfileMutation'; @@ -27,6 +25,7 @@ import SetProfileModal from 'components/indices/IndexSetFieldTypes/SetProfileMod import useProfileOptions from 'components/indices/IndexSetFieldTypeProfiles/hooks/useProfileOptions'; import useRemoveProfileFromIndexMutation from 'components/indices/IndexSetFieldTypes/hooks/useRemoveProfileFromIndexMutation'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; const selectItem = async (select: HTMLElement, option: string | RegExp) => { selectEvent.openMenu(select); @@ -35,9 +34,9 @@ const selectItem = async (select: HTMLElement, option: string | RegExp) => { }; const renderModal = (currentProfile = 'profile-id-111') => render( - + {}} show /> - , + , ); jest.mock('routing/useParams', () => jest.fn()); diff --git a/graylog2-web-interface/src/components/streams/StreamsOverview/StreamsOverview.test.tsx b/graylog2-web-interface/src/components/streams/StreamsOverview/StreamsOverview.test.tsx index becbc209569a..40fe20adb9ca 100644 --- a/graylog2-web-interface/src/components/streams/StreamsOverview/StreamsOverview.test.tsx +++ b/graylog2-web-interface/src/components/streams/StreamsOverview/StreamsOverview.test.tsx @@ -17,8 +17,6 @@ import React from 'react'; import { render, screen, within } from 'wrappedTestingLibrary'; import userEvent from '@testing-library/user-event'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; -import { QueryParamProvider } from 'use-query-params'; import { indexSets } from 'fixtures/indexSets'; import { asMock, MockStore } from 'helpers/mocking'; @@ -28,6 +26,7 @@ import useUserLayoutPreferences from 'components/common/EntityDataTable/hooks/us import { layoutPreferences } from 'fixtures/entityListLayoutPreferences'; import useStreamRuleTypes from 'components/streams/hooks/useStreamRuleTypes'; import { streamRuleTypes } from 'fixtures/streamRuleTypes'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; import StreamsOverview from './StreamsOverview'; @@ -77,9 +76,9 @@ const paginatedStreams = (exampleStream = stream) => ({ describe('StreamsOverview', () => { const renderSut = () => render( - + - , + , ); beforeEach(() => { diff --git a/graylog2-web-interface/src/pages/IndexSetFieldTypesPage.test.tsx b/graylog2-web-interface/src/pages/IndexSetFieldTypesPage.test.tsx index d14705a9a65a..07db968ae47f 100644 --- a/graylog2-web-interface/src/pages/IndexSetFieldTypesPage.test.tsx +++ b/graylog2-web-interface/src/pages/IndexSetFieldTypesPage.test.tsx @@ -16,8 +16,7 @@ */ import * as React from 'react'; import { render, screen, fireEvent, within } from 'wrappedTestingLibrary'; -import { useQueryParam, QueryParamProvider } from 'use-query-params'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; +import { useQueryParam } from 'use-query-params'; import { MockStore } from 'helpers/mocking'; import asMock from 'helpers/mocking/AsMock'; @@ -29,6 +28,7 @@ import useViewsPlugin from 'views/test/testViewsPlugin'; import IndexSetFieldTypesPage from 'pages/IndexSetFieldTypesPage'; import useFieldTypesForMappings from 'views/logic/fieldactions/ChangeFieldType/hooks/useFieldTypesForMappings'; import { overriddenIndexField, defaultField, attributes } from 'fixtures/indexSetFieldTypes'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; const getData = (list = [defaultField]) => ( { @@ -41,11 +41,11 @@ const getData = (list = [defaultField]) => ( ); const renderIndexSetFieldTypesPage = () => render( - + , - , + , ); jest.mock('views/logic/fieldactions/ChangeFieldType/hooks/useFieldTypesForMappings', () => jest.fn()); diff --git a/graylog2-web-interface/src/routing/App.tsx b/graylog2-web-interface/src/routing/App.tsx index 54aef0ed6b67..75b4ea518e4f 100644 --- a/graylog2-web-interface/src/routing/App.tsx +++ b/graylog2-web-interface/src/routing/App.tsx @@ -18,8 +18,6 @@ import React from 'react'; import styled, { css } from 'styled-components'; import chroma from 'chroma-js'; import { Outlet } from 'react-router-dom'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; -import { QueryParamProvider } from 'use-query-params'; import { ScratchpadProvider } from 'contexts/ScratchpadProvider'; import { Icon, Spinner } from 'components/common'; @@ -34,6 +32,7 @@ import HotkeysModalContainer from 'components/hotkeys/HotkeysModalContainer'; import PerspectivesProvider from 'components/perspectives/contexts/PerspectivesProvider'; import PageContextProviders from 'components/page/contexts/PageContextProviders'; import { singleton } from 'logic/singleton'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; const AppLayout = styled.div` display: flex; @@ -66,7 +65,7 @@ const ScrollToHint = styled.div(({ theme }) => css` `); const App = () => ( - + {(currentUser) => { if (!currentUser) { @@ -103,7 +102,7 @@ const App = () => ( ); }} - + ); export default singleton('components.App', () => App); diff --git a/graylog2-web-interface/src/routing/DefaultQueryParamProvider.tsx b/graylog2-web-interface/src/routing/DefaultQueryParamProvider.tsx new file mode 100644 index 000000000000..4c0c12719cee --- /dev/null +++ b/graylog2-web-interface/src/routing/DefaultQueryParamProvider.tsx @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import * as React from 'react'; +import type { PropsWithChildren } from 'react'; +import { QueryParamProvider } from 'use-query-params'; +import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; + +const DefaultQueryParamProvider = ({ children }: PropsWithChildren) => ( + + {children} + +); + +export default DefaultQueryParamProvider; diff --git a/graylog2-web-interface/src/views/components/dashboard/DashboardsOverview/DashboardsOverview.test.tsx b/graylog2-web-interface/src/views/components/dashboard/DashboardsOverview/DashboardsOverview.test.tsx index 847e5de0f18d..43a4866dfe1d 100644 --- a/graylog2-web-interface/src/views/components/dashboard/DashboardsOverview/DashboardsOverview.test.tsx +++ b/graylog2-web-interface/src/views/components/dashboard/DashboardsOverview/DashboardsOverview.test.tsx @@ -16,8 +16,7 @@ */ import React from 'react'; import { render, screen } from 'wrappedTestingLibrary'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; -import { useQueryParam, QueryParamProvider } from 'use-query-params'; +import { useQueryParam } from 'use-query-params'; import View from 'views/logic/views/View'; import Search from 'views/logic/search/Search'; @@ -25,6 +24,7 @@ import { asMock } from 'helpers/mocking'; import useFetchEntities from 'components/common/PaginatedEntityTable/useFetchEntities'; import useUserLayoutPreferences from 'components/common/EntityDataTable/hooks/useUserLayoutPreferences'; import { layoutPreferences } from 'fixtures/entityListLayoutPreferences'; +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; import DashboardsOverview from './DashboardsOverview'; @@ -98,9 +98,9 @@ const loadDashboardsResponse = (count = 1) => { describe('DashboardsOverview', () => { const SUT = () => ( - + - + ); beforeEach(() => { diff --git a/graylog2-web-interface/src/views/pages/DashboardsPage.test.tsx b/graylog2-web-interface/src/views/pages/DashboardsPage.test.tsx index ab129b31bf01..bd2a9cbfef9c 100644 --- a/graylog2-web-interface/src/views/pages/DashboardsPage.test.tsx +++ b/graylog2-web-interface/src/views/pages/DashboardsPage.test.tsx @@ -16,8 +16,8 @@ */ import * as React from 'react'; import { render, screen } from 'wrappedTestingLibrary'; -import { QueryParamProvider } from 'use-query-params'; -import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; + +import DefaultQueryParamProvider from 'routing/DefaultQueryParamProvider'; import DashboardsPage from './DashboardsPage'; @@ -41,9 +41,9 @@ jest.mock('routing/Routes', () => ({ describe('DashboardsPage', () => { it('should render header and list', async () => { render( - + - ); + ); await screen.findByRole('heading', { name: /dashboards/i }); await screen.findByText('No dashboards have been found.'); From 8aafad6822d43021bb47b3bb84223e96a2243612 Mon Sep 17 00:00:00 2001 From: Jan Heise Date: Thu, 19 Dec 2024 09:13:58 +0100 Subject: [PATCH 10/12] Filter system event definitions from content pack generation (#21206) * filters system event definitions from content pack generation * adding changelog --- changelog/unreleased/pr-21206.toml | 5 +++++ .../SystemNotificationEventProcessorConfig.java | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 changelog/unreleased/pr-21206.toml diff --git a/changelog/unreleased/pr-21206.toml b/changelog/unreleased/pr-21206.toml new file mode 100644 index 000000000000..dac3a9f2ffc7 --- /dev/null +++ b/changelog/unreleased/pr-21206.toml @@ -0,0 +1,5 @@ +type = "f" +message = "Filters system event definitions from list of entities that can be exported in a content pack." + +pulls = ["21206"] +issues = ["21166"] diff --git a/graylog2-server/src/main/java/org/graylog/events/processor/systemnotification/SystemNotificationEventProcessorConfig.java b/graylog2-server/src/main/java/org/graylog/events/processor/systemnotification/SystemNotificationEventProcessorConfig.java index 583db1dadf61..3e73e42c81aa 100644 --- a/graylog2-server/src/main/java/org/graylog/events/processor/systemnotification/SystemNotificationEventProcessorConfig.java +++ b/graylog2-server/src/main/java/org/graylog/events/processor/systemnotification/SystemNotificationEventProcessorConfig.java @@ -62,6 +62,11 @@ public EventProcessorConfigEntity toContentPackEntity(EntityDescriptorIds entity return null; } + @Override + public boolean isContentPackExportable() { + return false; + } + @Override public void resolveNativeEntity(EntityDescriptor entityDescriptor, MutableGraph mutableGraph) { } From 7664ddf6a373ca526c69435c254e32ccd145e786 Mon Sep 17 00:00:00 2001 From: Anton Ebel Date: Thu, 19 Dec 2024 14:22:04 +0100 Subject: [PATCH 11/12] add archive restore retry on mapper parsing exception (#21197) * add restore retry on mapper parsing exception * add changelog * fix changelog * update changelog Co-authored-by: Patrick Mann --------- Co-authored-by: Patrick Mann --- changelog/unreleased/pr-21197.toml | 5 ++++ .../elasticsearch7/ElasticsearchClient.java | 8 +++++++ .../storage/opensearch2/OpenSearchClient.java | 8 +++++++ .../indexer/MapperParsingException.java | 23 +++++++++++++++++++ .../org/graylog2/indexer/indices/Indices.java | 9 ++++++-- 5 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/pr-21197.toml create mode 100644 graylog2-server/src/main/java/org/graylog2/indexer/MapperParsingException.java diff --git a/changelog/unreleased/pr-21197.toml b/changelog/unreleased/pr-21197.toml new file mode 100644 index 000000000000..89604ad1f796 --- /dev/null +++ b/changelog/unreleased/pr-21197.toml @@ -0,0 +1,5 @@ +type = "fixed" +message = "Add archive restore retry on mapper parsing exception." + +issues = ["graylog-plugin-enterprise#9208"] +pulls = ["21197", "Graylog2/graylog-plugin-enterprise#9413"] diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ElasticsearchClient.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ElasticsearchClient.java index c6c45d1f2de8..4f16148a0cb6 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ElasticsearchClient.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ElasticsearchClient.java @@ -42,6 +42,7 @@ import org.graylog2.indexer.BatchSizeTooLargeException; import org.graylog2.indexer.IndexNotFoundException; import org.graylog2.indexer.InvalidWriteTargetException; +import org.graylog2.indexer.MapperParsingException; import org.graylog2.indexer.MasterNotDiscoveredException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -202,6 +203,9 @@ private ElasticsearchException exceptionFrom(Exception e, String errorMessage) { if (isBatchSizeTooLargeException(elasticsearchException)) { throw new BatchSizeTooLargeException(elasticsearchException.getMessage()); } + if (isMapperParsingExceptionException(elasticsearchException)) { + throw new MapperParsingException(elasticsearchException.getMessage()); + } } else if (e instanceof IOException && e.getCause() instanceof ContentTooLongException) { throw new BatchSizeTooLargeException(e.getMessage()); } @@ -231,6 +235,10 @@ private boolean isIndexNotFoundException(ElasticsearchException elasticsearchExc return elasticsearchException.getMessage().contains("index_not_found_exception"); } + private boolean isMapperParsingExceptionException(ElasticsearchException openSearchException) { + return openSearchException.getMessage().contains("mapper_parsing_exception"); + } + private boolean isBatchSizeTooLargeException(ElasticsearchException elasticsearchException) { if (elasticsearchException instanceof ElasticsearchStatusException statusException) { if (statusException.getCause() instanceof ResponseException responseException) { diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/OpenSearchClient.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/OpenSearchClient.java index e12b820a3a14..0bc939e48e6d 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/OpenSearchClient.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/OpenSearchClient.java @@ -42,6 +42,7 @@ import org.graylog2.indexer.BatchSizeTooLargeException; import org.graylog2.indexer.IndexNotFoundException; import org.graylog2.indexer.InvalidWriteTargetException; +import org.graylog2.indexer.MapperParsingException; import org.graylog2.indexer.MasterNotDiscoveredException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -202,6 +203,9 @@ private OpenSearchException exceptionFrom(Exception e, String errorMessage) { if (isBatchSizeTooLargeException(openSearchException)) { throw new BatchSizeTooLargeException(openSearchException.getMessage()); } + if (isMapperParsingExceptionException(openSearchException)) { + throw new MapperParsingException(openSearchException.getMessage()); + } } else if (e instanceof IOException && e.getCause() instanceof ContentTooLongException) { throw new BatchSizeTooLargeException(e.getMessage()); } @@ -231,6 +235,10 @@ private boolean isIndexNotFoundException(OpenSearchException openSearchException return openSearchException.getMessage().contains("index_not_found_exception"); } + private boolean isMapperParsingExceptionException(OpenSearchException openSearchException) { + return openSearchException.getMessage().contains("mapper_parsing_exception"); + } + private boolean isBatchSizeTooLargeException(OpenSearchException openSearchException) { if (openSearchException instanceof OpenSearchStatusException statusException) { if (statusException.getCause() instanceof ResponseException responseException) { diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/MapperParsingException.java b/graylog2-server/src/main/java/org/graylog2/indexer/MapperParsingException.java new file mode 100644 index 000000000000..e7c656adf015 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/indexer/MapperParsingException.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.indexer; + +public class MapperParsingException extends ElasticsearchException { + public MapperParsingException(String errorMessage) { + super(errorMessage); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java index 5dd979d9a055..1a98434c011c 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java @@ -36,6 +36,7 @@ import org.graylog2.indexer.IndexNotFoundException; import org.graylog2.indexer.IndexSet; import org.graylog2.indexer.IndexTemplateNotFoundException; +import org.graylog2.indexer.MapperParsingException; import org.graylog2.indexer.indexset.CustomFieldMappings; import org.graylog2.indexer.indexset.IndexSetConfig; import org.graylog2.indexer.indexset.IndexSetMappingTemplate; @@ -227,7 +228,7 @@ public void deleteIndexTemplate(IndexSet indexSet) { } public boolean create(String indexName, IndexSet indexSet) { - return create(indexName, indexSet, null, null ); + return create(indexName, indexSet, null, null); } public boolean create(String indexName, @@ -248,6 +249,10 @@ public boolean create(String indexName, indicesAdapter.create(indexName, settings, mappings); } catch (Exception e) { + if ((indexSettings != null || indexMapping != null) && e instanceof MapperParsingException) { + LOG.info("Couldn't create index {}. Error: {}. Fall back to default settings/mappings and retry.", indexName, e.getMessage(), e); + return create(indexName, indexSet, null, null); + } LOG.warn("Couldn't create index {}. Error: {}", indexName, e.getMessage(), e); auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_CREATE, ImmutableMap.of("indexName", indexName)); return false; @@ -259,7 +264,7 @@ public boolean create(String indexName, private Optional indexMapping(IndexSet indexSet) { try { return Optional.of(indexMappingFactory.createIndexMapping(indexSet.getConfig())); - }catch (IgnoreIndexTemplate e){ + } catch (IgnoreIndexTemplate e) { return Optional.empty(); } } From 7bc444d4ddfd1e631686833194e837dc5f8dbc5e Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Thu, 19 Dec 2024 14:44:18 +0100 Subject: [PATCH 12/12] Allow custom filter components for paginated data table. (#21216) * Allow custom filter components for paginated data table. * Adding test cases for custom filter input component. * Adding license headers. --- .../EntityFilters/EntityFilters.test.tsx | 82 ++++++++++++++++++- .../FilterConfiguration.tsx | 41 +++++----- .../FilterConfiguration/SuggestionsList.tsx | 46 ++++++----- .../SuggestionsListFilter.tsx | 59 +++++++++++++ .../helpers/AttributeIdentification.ts | 1 + .../src/stores/PaginationTypes.ts | 19 +++-- 6 files changed, 201 insertions(+), 47 deletions(-) create mode 100644 graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/SuggestionsListFilter.tsx diff --git a/graylog2-web-interface/src/components/common/EntityFilters/EntityFilters.test.tsx b/graylog2-web-interface/src/components/common/EntityFilters/EntityFilters.test.tsx index d185e96945e8..fd9c55ffac16 100644 --- a/graylog2-web-interface/src/components/common/EntityFilters/EntityFilters.test.tsx +++ b/graylog2-web-interface/src/components/common/EntityFilters/EntityFilters.test.tsx @@ -19,11 +19,13 @@ import { render, screen, waitFor, within } from 'wrappedTestingLibrary'; import userEvent from '@testing-library/user-event'; import { OrderedMap } from 'immutable'; import type { Optional } from 'utility-types'; +import { Formik, Form } from 'formik'; -import type { Attributes } from 'stores/PaginationTypes'; +import type { Attributes, FilterComponentProps } from 'stores/PaginationTypes'; import { asMock } from 'helpers/mocking'; import useFilterValueSuggestions from 'components/common/EntityFilters/hooks/useFilterValueSuggestions'; import useFiltersWithTitle from 'components/common/EntityFilters/hooks/useFiltersWithTitle'; +import { ModalSubmit, FormikInput } from 'components/common'; import OriginalEntityFilters from './EntityFilters'; @@ -36,10 +38,31 @@ jest.mock('logic/generateId', () => jest.fn(() => 'filter-id')); jest.mock('components/common/EntityFilters/hooks/useFilterValueSuggestions'); jest.mock('components/common/EntityFilters/hooks/useFiltersWithTitle'); +const CustomFilterInput = ({ filter, onSubmit }: FilterComponentProps) => ( +
+ onSubmit({ title: value, value })}> + {({ isValid }) => ( +
+ + + + )} +
+
+); + describe('', () => { const onChangeFiltersWithTitle = jest.fn(); const setUrlQueryFilters = jest.fn(); - const attributes = [ + const attributes: Attributes = [ { id: 'title', title: 'Title', sortable: true }, { id: 'description', title: 'Description', sortable: true }, { @@ -88,7 +111,14 @@ describe('', () => { title: 'Generic Attribute', type: 'STRING', }, - ] as Attributes; + { + id: 'customComponent', + filterable: true, + title: 'Custom Component Attribute', + type: 'STRING', + filter_component: CustomFilterInput, + }, + ]; const EntityFilters = (props: Optional, 'setUrlQueryFilters' | 'attributes'>) => ( @@ -424,6 +454,52 @@ describe('', () => { }); }); + describe('custom component attribute', () => { + it('provides text input to create filter', async () => { + render( + , + ); + + userEvent.click(await screen.findByRole('button', { name: /create filter/i })); + + userEvent.click(await screen.findByRole('menuitem', { name: /custom component/i })); + + const filterInput = await screen.findByPlaceholderText('My custom input'); + userEvent.type(filterInput, 'foo'); + + const form = await screen.findByTestId('custom-component-form'); + userEvent.click(await within(form).findByRole('button', { name: /create filter/i })); + + await waitFor(() => { + expect(setUrlQueryFilters).toHaveBeenCalledWith(OrderedMap({ customComponent: ['foo'] })); + }); + }); + + it('allows changing filter', async () => { + asMock(useFiltersWithTitle).mockReturnValue({ + data: OrderedMap({ customComponent: [{ title: 'foo', value: 'foo' }] }), + onChange: onChangeFiltersWithTitle, + isInitialLoading: false, + }); + + render( + , + ); + + userEvent.click(await screen.findByText('foo')); + + const filterInput = await screen.findByPlaceholderText('My custom input'); + userEvent.type(filterInput, '{selectall}bar'); + + const form = await screen.findByTestId('custom-component-form'); + userEvent.click(await within(form).findByRole('button', { name: /update filter/i })); + + await waitFor(() => { + expect(setUrlQueryFilters).toHaveBeenCalledWith(OrderedMap({ customComponent: ['bar'] })); + }); + }); + }); + it('should display active filters', async () => { asMock(useFiltersWithTitle).mockReturnValue({ data: OrderedMap({ diff --git a/graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/FilterConfiguration.tsx b/graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/FilterConfiguration.tsx index 7919cea70546..2a382c74954e 100644 --- a/graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/FilterConfiguration.tsx +++ b/graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/FilterConfiguration.tsx @@ -16,28 +16,31 @@ */ import * as React from 'react'; -import type { Attribute } from 'stores/PaginationTypes'; -import type { Filters, Filter } from 'components/common/EntityFilters/types'; +import type { FilterComponentProps } from 'stores/PaginationTypes'; import { MenuItem } from 'components/bootstrap'; import { isAttributeWithFilterOptions, - isAttributeWithRelatedCollection, isDateAttribute, + isAttributeWithRelatedCollection, isDateAttribute, isCustomComponentFilter, } from 'components/common/EntityFilters/helpers/AttributeIdentification'; -import GenericFilterInput from 'components/common/EntityFilters/FilterConfiguration/GenericFilterInput'; +import SuggestionsListFilter from './SuggestionsListFilter'; +import GenericFilterInput from './GenericFilterInput'; import StaticOptionsList from './StaticOptionsList'; -import SuggestionsList from './SuggestionsList'; import DateRangeForm from './DateRangeForm'; -type Props = { - attribute: Attribute, - filter?: Filter, - filterValueRenderer: (value: Filter['value'], title: string) => React.ReactNode | undefined, - onSubmit: (filter: { title: string, value: string }, closeDropdown?: boolean) => void, - allActiveFilters: Filters | undefined, -} +const FilterComponent = ({ allActiveFilters, attribute, filter = undefined, filterValueRenderer, onSubmit }: FilterComponentProps) => { + if (isCustomComponentFilter(attribute)) { + const CustomFilterComponent = attribute.filter_component; + + return ( + + ); + } -const FilterComponent = ({ allActiveFilters, attribute, filter, filterValueRenderer, onSubmit }: Pick) => { if (isAttributeWithFilterOptions(attribute)) { return ( + ); } @@ -73,7 +76,7 @@ export const FilterConfiguration = ({ filter = undefined, filterValueRenderer, onSubmit, -}: Props) => ( +}: FilterComponentProps) => ( <> {filter ? 'Edit' : 'Create'} {attribute.title.toLowerCase()} filter . */ -import React, { useState, useCallback } from 'react'; +import React, { useCallback } from 'react'; import debounce from 'lodash/debounce'; import styled, { css } from 'styled-components'; @@ -23,15 +23,8 @@ import type { Attribute } from 'stores/PaginationTypes'; import type { Filters, Filter } from 'components/common/EntityFilters/types'; import { PaginatedList, NoSearchResult } from 'components/common'; import useIsKeyHeld from 'hooks/useIsKeyHeld'; -import useFilterValueSuggestions from 'components/common/EntityFilters/hooks/useFilterValueSuggestions'; import Spinner from 'components/common/Spinner'; -const DEFAULT_SEARCH_PARAMS = { - query: '', - pageSize: 10, - page: 1, -}; - const Container = styled.div(({ theme }) => css` color: ${theme.colors.global.textDefault}; padding: 3px 10px; @@ -50,25 +43,40 @@ const Hint = styled.div(({ theme }) => css` font-size: ${theme.fonts.size.small}; `); +type SearchParams = { + query: string, + page: number, + pageSize: number, +} + +type Suggestion = { + id: string, + value: string, +} + type Props = { allActiveFilters: Filters | undefined, attribute: Attribute, filter: Filter | undefined filterValueRenderer: (value: unknown, title: string) => React.ReactNode | undefined, onSubmit: (filter: { title: string, value: string }, closeDropdown: boolean) => void, + suggestions: Array, + isLoading: boolean, + total: number, + page: number, + pageSize: number, + setSearchParams: (updater: (current: SearchParams) => SearchParams) => void, } -const SuggestionsList = ({ attribute, filterValueRenderer, onSubmit, allActiveFilters, filter }: Props) => { +const SuggestionsList = ({ attribute, filterValueRenderer, onSubmit, allActiveFilters, filter, isLoading, suggestions, total, setSearchParams, page, pageSize }: Props) => { const isShiftHeld = useIsKeyHeld('Shift'); - const [searchParams, setSearchParams] = useState(DEFAULT_SEARCH_PARAMS); - const { data: { pagination, suggestions }, isInitialLoading } = useFilterValueSuggestions(attribute.id, attribute.related_collection, searchParams, attribute.related_property); const handleSearchChange = useCallback((newSearchQuery: string) => { - setSearchParams((cur) => ({ ...cur, page: DEFAULT_SEARCH_PARAMS.page, query: newSearchQuery })); + setSearchParams((cur) => ({ ...cur, page: 1, query: newSearchQuery })); }, [setSearchParams]); - const handlePaginationChange = useCallback((page: number) => { - setSearchParams((cur) => ({ ...cur, page })); - }, []); + const handlePaginationChange = useCallback((newPage: number) => { + setSearchParams((cur) => ({ ...cur, page: newPage })); + }, [setSearchParams]); const debounceOnSearch = debounce((value: string) => handleSearchChange(value), 1000); @@ -79,15 +87,15 @@ const SuggestionsList = ({ attribute, filterValueRenderer, onSubmit, allActiveFi formGroupClassName="" placeholder={`Search for ${attribute.title.toLowerCase()}`} onChange={({ target: { value } }) => debounceOnSearch(value)} /> - {isInitialLoading && } + {isLoading && } {!!suggestions?.length && ( diff --git a/graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/SuggestionsListFilter.tsx b/graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/SuggestionsListFilter.tsx new file mode 100644 index 000000000000..eb3e70961d6a --- /dev/null +++ b/graylog2-web-interface/src/components/common/EntityFilters/FilterConfiguration/SuggestionsListFilter.tsx @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import * as React from 'react'; +import { useState } from 'react'; + +import type { Filters, Filter } from 'components/common/EntityFilters/types'; +import type { Attribute } from 'stores/PaginationTypes'; +import useFilterValueSuggestions from 'components/common/EntityFilters/hooks/useFilterValueSuggestions'; + +import SuggestionsList from './SuggestionsList'; + +type Props = { + allActiveFilters: Filters | undefined, + attribute: Attribute, + filter: Filter | undefined + filterValueRenderer: (value: unknown, title: string) => React.ReactNode | undefined, + onSubmit: (filter: { title: string, value: string }, closeDropdown: boolean) => void, +} + +const DEFAULT_SEARCH_PARAMS = { + query: '', + pageSize: 10, + page: 1, +}; + +const SuggestionsListFilter = ({ attribute, filterValueRenderer, onSubmit, allActiveFilters, filter }: Props) => { + const [searchParams, setSearchParams] = useState(DEFAULT_SEARCH_PARAMS); + const { data: { pagination, suggestions }, isInitialLoading } = useFilterValueSuggestions(attribute.id, attribute.related_collection, searchParams, attribute.related_property); + + return ( + + ); +}; + +export default SuggestionsListFilter; diff --git a/graylog2-web-interface/src/components/common/EntityFilters/helpers/AttributeIdentification.ts b/graylog2-web-interface/src/components/common/EntityFilters/helpers/AttributeIdentification.ts index 8c27e129df55..18564b17b799 100644 --- a/graylog2-web-interface/src/components/common/EntityFilters/helpers/AttributeIdentification.ts +++ b/graylog2-web-interface/src/components/common/EntityFilters/helpers/AttributeIdentification.ts @@ -19,3 +19,4 @@ import type { Attribute } from 'stores/PaginationTypes'; export const isDateAttribute = ({ type }: Attribute) => type === 'DATE'; export const isAttributeWithFilterOptions = ({ filter_options }: Attribute) => !!filter_options?.length; export const isAttributeWithRelatedCollection = ({ related_collection }: Attribute) => !!related_collection; +export const isCustomComponentFilter = ({ filter_component }: Attribute) => !!filter_component; diff --git a/graylog2-web-interface/src/stores/PaginationTypes.ts b/graylog2-web-interface/src/stores/PaginationTypes.ts index 4b9b508440c2..f1ff74af028d 100644 --- a/graylog2-web-interface/src/stores/PaginationTypes.ts +++ b/graylog2-web-interface/src/stores/PaginationTypes.ts @@ -15,10 +15,9 @@ * . */ import type * as Immutable from 'immutable'; -import type { $PropertyType } from 'utility-types'; import type { AdditionalQueries } from 'util/PaginationURL'; -import type { UrlQueryFilters } from 'components/common/EntityFilters/types'; +import type { UrlQueryFilters, Filter, Filters } from 'components/common/EntityFilters/types'; export type PaginatedResponseType = { count: number, @@ -29,9 +28,9 @@ export type PaginatedResponseType = { }; export type PaginatedListJSON = { - page: $PropertyType, - per_page: $PropertyType, - query: $PropertyType, + page: Pagination['page'], + per_page: Pagination['perPage'], + query: Pagination['query'], total: number, count: number, }; @@ -72,6 +71,13 @@ export type SearchParams = { filters?: UrlQueryFilters } +export type FilterComponentProps = { + attribute: Attribute, + filter?: Filter, + filterValueRenderer: (value: Filter['value'], title: string) => React.ReactNode | undefined, + onSubmit: (filter: { title: string, value: string }, closeDropdown?: boolean) => void, + allActiveFilters: Filters | undefined, +} export type Attribute = { id: string, title: string, @@ -80,7 +86,8 @@ export type Attribute = { hidden?: boolean, searchable?: boolean, filterable?: true, - filter_options?: Array<{ value: string, title: string }> + filter_options?: Array<{ value: string, title: string }>, + filter_component?: React.ComponentType, related_collection?: string, related_property?: string, permissions?: Array,