From 3657b944c2028ac113fc6abd5971fb26c50c44d7 Mon Sep 17 00:00:00 2001 From: Patrick Mann Date: Tue, 14 Nov 2023 12:20:56 +0100 Subject: [PATCH 1/2] Add CRC to sidecar default template (#17246) * add CRC for default template * add hardcode prior version CRCs * update CL * Only update if different * log if ignoring an unknown, old template * more legacy CRCs * review feedback fixes * handle PUT and POST correctly * remove redundant CRC in request --- changelog/unreleased/issue-17040.toml | 5 ++ .../V20180212165000_AddDefaultCollectors.java | 23 +++++- .../sidecar/rest/models/Collector.java | 71 ++++++++++++++++++- .../rest/resources/CollectorResource.java | 21 +++--- .../sidecar/services/CollectorService.java | 3 +- .../collectors/SidecarServiceTest.java | 2 +- 6 files changed, 110 insertions(+), 15 deletions(-) create mode 100644 changelog/unreleased/issue-17040.toml diff --git a/changelog/unreleased/issue-17040.toml b/changelog/unreleased/issue-17040.toml new file mode 100644 index 000000000000..5667a6ecf705 --- /dev/null +++ b/changelog/unreleased/issue-17040.toml @@ -0,0 +1,5 @@ +type = "c" +message = "Enable sidecar default configurations to be updated on existing installs." + +issues = ["17040"] +pulls = ["17246"] diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java index ccec07be882a..7fe7d155848e 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java @@ -35,6 +35,7 @@ import javax.inject.Inject; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.time.ZonedDateTime; import java.util.Optional; import java.util.Set; @@ -81,7 +82,6 @@ public ZonedDateTime createdAt() { @Override public void upgrade() { - removeConfigPath(); ensureConfigurationVariable("graylog_host", "Graylog Host.", httpExternalUri.getHost()); @@ -503,6 +503,23 @@ private Optional ensureCollector(String collectorName, LOG.debug(msg, collectorName, nodeOperatingSystem); throw new IllegalArgumentException(); } + if (!collector.defaultTemplateUpdated()) { + long newCRC = Collector.checksum(defaultTemplate.getBytes(StandardCharsets.UTF_8)); + if (collector.defaultTemplateCRC() == null // known obsolete version of template + || newCRC != collector.defaultTemplateCRC() // new standard template + ) { + LOG.info("{} collector default template on {} is unchanged, updating it.", collectorName, nodeOperatingSystem); + try { + return Optional.of(collectorService.save( + collector.toBuilder() + .defaultTemplate(defaultTemplate) + .defaultTemplateCRC(newCRC) + .build())); + } catch (Exception e) { + LOG.error("Can't save collector '{}'!", collectorName, e); + } + } + } } catch (IllegalArgumentException ignored) { LOG.info("{} collector on {} is missing, adding it.", collectorName, nodeOperatingSystem); try { @@ -514,7 +531,8 @@ private Optional ensureCollector(String collectorName, executablePath, executeParameters, validationCommand, - defaultTemplate + defaultTemplate, + Collector.checksum(defaultTemplate.getBytes(StandardCharsets.UTF_8)) ))); } catch (Exception e) { LOG.error("Can't save collector '{}'!", collectorName, e); @@ -577,5 +595,4 @@ private void ensureDefaultConfiguration(String name, Collector collector) { LOG.error("Unable to access '{}' sidecar default configuration!", name); } } - } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Collector.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Collector.java index 499d3ede5ca0..17c5bfaf5e72 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Collector.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Collector.java @@ -16,16 +16,27 @@ */ package org.graylog.plugins.sidecar.rest.models; -import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; import org.mongojack.Id; import org.mongojack.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; +import java.nio.charset.StandardCharsets; +import java.util.Set; +import java.util.zip.CRC32; +import java.util.zip.Checksum; @AutoValue @JsonAutoDetect public abstract class Collector { + private static final Logger LOG = LoggerFactory.getLogger(Collector.class); + public static final String FIELD_ID = "id"; public static final String FIELD_NAME = "name"; public static final String FIELD_SERVICE_TYPE = "service_type"; @@ -34,6 +45,30 @@ public abstract class Collector { public static final String FIELD_EXECUTE_PARAMETERS = "execute_parameters"; public static final String FIELD_VALIDATION_PARAMETERS = "validation_parameters"; public static final String FIELD_DEFAULT_TEMPLATE = "default_template"; + public static final String FIELD_DEFAULT_TEMPLATE_CRC = "default_template_crc"; + + // Set of prior version CRCs for back-compat + private static final Set INITIAL_CRC = java.util.Set.of( + 3280545580L, // 5.2 filebeat linux + 3396210381L, // 5.2 filebeat darwin + 3013497446L, // 5.2 filebeat freebsd + 4009863009L, // 5.2 winlogbeat windows + 2023247173L, // 5.2 nxlog linux + 2491201449L, // 5.2 nxlog windows + 2487909285L, // 5.2 auditbeat windows + + 4049210961L, // 5.1 and 5.0 filebeat linux/darwin/freebsd + 2306685777L, // 5.1 and 5.0 winlogbeat windows + 639836274L, // 5.1 and 5.0 nxlog linux + 2157898695L, // 5.1 and 5.0 nxlog windows + 1490581247L, // 5.1 and 5.0 filebeat windows + + 1256873081L, // 4.3 filebeat linux + 3852098581L, // 4.3 winlogbeat windows + 3676599312L, // 4.3 nxlog linux + 4293222217L, // 4.3 nxlog windows + 2559816928L // 4.3 filebeat windows + ); @Id @ObjectId @@ -66,6 +101,35 @@ public abstract class Collector { @Nullable public abstract String defaultTemplate(); + @JsonProperty(FIELD_DEFAULT_TEMPLATE_CRC) + @Nullable + public abstract Long defaultTemplateCRC(); + + @JsonIgnore + public boolean defaultTemplateUpdated() { + if (defaultTemplate() == null) { + return false; + } + + long crc = checksum(defaultTemplate().getBytes(StandardCharsets.UTF_8)); + if (defaultTemplateCRC() == null) { + if (INITIAL_CRC.contains(crc)) { + return false; // known old version + } else { + LOG.info("{} collector default template on {} is an unrecognized version - not updating automatically.", name(), nodeOperatingSystem()); + return true; // changed or really old standard default template + } + } + return (crc != defaultTemplateCRC()); + } + + @JsonIgnore + public static long checksum(byte[] bytes) { + Checksum crc32 = new CRC32(); + crc32.update(bytes, 0, bytes.length); + return crc32.getValue(); + } + public static Builder builder() { return new AutoValue_Collector.Builder(); } @@ -82,6 +146,7 @@ public abstract static class Builder { public abstract Builder executeParameters(String executeParameters); public abstract Builder validationParameters(String validationParameters); public abstract Builder defaultTemplate(String defaultTemplate); + public abstract Builder defaultTemplateCRC(Long checksum); public abstract Collector build(); } @@ -93,7 +158,8 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id, @JsonProperty(FIELD_EXECUTABLE_PATH) String executablePath, @JsonProperty(FIELD_EXECUTE_PARAMETERS) @Nullable String executeParameters, @JsonProperty(FIELD_VALIDATION_PARAMETERS) @Nullable String validationParameters, - @JsonProperty(FIELD_DEFAULT_TEMPLATE) @Nullable String defaultTemplate) { + @JsonProperty(FIELD_DEFAULT_TEMPLATE) @Nullable String defaultTemplate, + @JsonProperty(FIELD_DEFAULT_TEMPLATE_CRC) @Nullable Long defaultTemplateCRC) { return builder() .id(id) .name(name) @@ -103,6 +169,7 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id, .executeParameters(executeParameters) .validationParameters(validationParameters) .defaultTemplate(defaultTemplate) + .defaultTemplateCRC(defaultTemplateCRC) .build(); } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/CollectorResource.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/CollectorResource.java index 517e4ed25076..a8c134b9999e 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/CollectorResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/CollectorResource.java @@ -204,13 +204,7 @@ public CollectorSummaryResponse listSummary(@ApiParam(name = "page") @QueryParam @AuditEvent(type = SidecarAuditEventTypes.COLLECTOR_CREATE) public Response createCollector(@ApiParam(name = "JSON body", required = true) @Valid @NotNull Collector request) throws BadRequestException { - Collector collector = collectorService.fromRequest(request); - final ValidationResult validationResult = validate(collector); - if (validationResult.failed()) { - return Response.status(Response.Status.BAD_REQUEST).entity(validationResult).build(); - } - etagService.invalidateAllCollectors(); - return Response.ok().entity(collectorService.save(collector)).build(); + return saveCollector(collectorService.fromRequest(request)); } @PUT @@ -223,11 +217,22 @@ public Response updateCollector(@ApiParam(name = "id", required = true) @PathParam("id") String id, @ApiParam(name = "JSON body", required = true) @Valid @NotNull Collector request) throws BadRequestException { - Collector collector = collectorService.fromRequest(id, request); + return saveCollector(collectorService.fromRequest(id, request)); + } + + private Response saveCollector(Collector collector) { final ValidationResult validationResult = validate(collector); if (validationResult.failed()) { return Response.status(Response.Status.BAD_REQUEST).entity(validationResult).build(); } + + // Don't overwrite CRC of an existing collector. We need the original value to + // know that the entry has been modified. + Collector existingCollector = collectorService.findByNameAndOs(collector.name(), collector.nodeOperatingSystem()); + if (existingCollector != null && existingCollector.defaultTemplateCRC() != null) { + collector = collector.toBuilder().defaultTemplateCRC(existingCollector.defaultTemplateCRC()).build(); + } + etagService.invalidateAllCollectors(); return Response.ok().entity(collectorService.save(collector)).build(); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/CollectorService.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/CollectorService.java index be8a09542f3b..c67d80c80527 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/CollectorService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/CollectorService.java @@ -101,7 +101,8 @@ public Collector fromRequest(Collector request) { request.executablePath(), request.executeParameters(), request.validationParameters(), - request.defaultTemplate()); + request.defaultTemplate(), + null); // this is only ever written by 20180212165000_AddDefaultCollectors } public Collector fromRequest(String id, Collector request) { diff --git a/graylog2-server/src/test/java/org/graylog/plugins/sidecar/collectors/SidecarServiceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/sidecar/collectors/SidecarServiceTest.java index c137eb89de30..11685380a20f 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/sidecar/collectors/SidecarServiceTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/sidecar/collectors/SidecarServiceTest.java @@ -365,7 +365,7 @@ private static Configuration getConfiguration() { private static Collector getCollector() { return Collector.create("collector-id", "collector-name", "service", "linux", - "/path", "param", "valid param", ""); + "/path", "param", "valid param", "", null); } private Sidecar getTestSidecar() { From df0e7e3422d2af366019f709b28a235f6e5c86dd Mon Sep 17 00:00:00 2001 From: maxiadlovskii Date: Tue, 14 Nov 2023 15:40:56 +0100 Subject: [PATCH 2/2] Add logic to change perPage in component ListWithOwnState states when new perPage prop arrives (#17278) * Add logic to change perPage in state of ListWithOwnState component when new perPage prop would arrive * Some refactoring * Add changelog * fix changelog * fix changelog * fix changelog --- changelog/unreleased/issue-17261.toml | 4 ++++ .../src/components/common/PaginatedList.tsx | 23 ++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/issue-17261.toml diff --git a/changelog/unreleased/issue-17261.toml b/changelog/unreleased/issue-17261.toml new file mode 100644 index 000000000000..4d799864908f --- /dev/null +++ b/changelog/unreleased/issue-17261.toml @@ -0,0 +1,4 @@ +type = "f" +message = "Fixing problem with persisting selected page size for some paginated lists." +issues = ["17261"] +pulls = ["17278"] diff --git a/graylog2-web-interface/src/components/common/PaginatedList.tsx b/graylog2-web-interface/src/components/common/PaginatedList.tsx index be8ddecff8c7..c94a86fa38d4 100644 --- a/graylog2-web-interface/src/components/common/PaginatedList.tsx +++ b/graylog2-web-interface/src/components/common/PaginatedList.tsx @@ -15,7 +15,7 @@ * . */ import * as React from 'react'; -import { useEffect, useMemo, useCallback } from 'react'; +import { useEffect, useMemo, useCallback, useState } from 'react'; import IfInteractive from 'views/components/dashboard/IfInteractive'; import usePaginationQueryParameter, { DEFAULT_PAGE_SIZES } from 'hooks/usePaginationQueryParameter'; @@ -110,16 +110,23 @@ const ListWithOwnState = ({ pageSize: propPageSize, ...props }: Required & { activePage: number, pageSize: number }) => { - const [{ page: currentPage, pageSize: currentPageSize }, setPagination] = React.useState({ - page: Math.max(activePage, INITIAL_PAGE), - pageSize: propPageSize, - }); + const [currentPage, setCurrentPage] = useState(Math.max(activePage, INITIAL_PAGE)); + const [currentPageSize, setCurrentPageSize] = useState(propPageSize); useEffect(() => { - if (activePage > 0 && activePage !== currentPage) { - setPagination((cur) => ({ ...cur, page: activePage })); + if (activePage > 0) { + setCurrentPage(activePage); } - }, [activePage, currentPage]); + }, [activePage]); + + useEffect(() => { + setCurrentPageSize(propPageSize); + }, [propPageSize]); + + const setPagination = useCallback(({ page, pageSize }) => { + setCurrentPageSize(pageSize); + setCurrentPage(page); + }, []); return (