diff --git a/UPGRADING.md b/UPGRADING.md index 58461c0f296d..0ded98b49637 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -23,9 +23,9 @@ Alternatively the `SaveOrCancelButtons` component can be used in the edit compon The following Java Code API changes have been made. -| File/method | Description | -|-------------------------------------|------------------------| -| `tbd` | tbd | +| File/method | Description | +|------------------------------------------------|-------------| +| `org.graylog.scheduler.JobSchedule#toDBUpdate` | removed | ## REST API Endpoint Changes diff --git a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java index ebd0c16dc65d..a99dd53abad7 100644 --- a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java +++ b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java @@ -29,11 +29,12 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.Codec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTime; import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import java.util.Optional; public class CloudTrailCodec extends AbstractCodec { public static final String NAME = "AWSCloudTrail"; @@ -49,9 +50,8 @@ public CloudTrailCodec(@Assisted Configuration configuration, @AWSObjectMapper O this.messageFactory = messageFactory; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { try { final CloudTrailRecord record = objectMapper.readValue(rawMessage.getPayload(), CloudTrailRecord.class); final String source = configuration.getString(Config.CK_OVERRIDE_SOURCE, "aws-cloudtrail"); @@ -61,9 +61,10 @@ public Message decode(@Nonnull RawMessage rawMessage) { message.addField("full_message", record.getFullMessage()); message.addField(AWS.SOURCE_GROUP_IDENTIFIER, true); - return message; + return Optional.of(message); } catch (Exception e) { - throw new RuntimeException("Could not deserialize CloudTrail record.", e); + throw InputProcessingException.create("Could not deserialize CloudTrail record.", + e, rawMessage, new String(rawMessage.getPayload(), charset)); } } diff --git a/graylog2-server/src/main/java/org/graylog/events/context/EventDefinitionContextService.java b/graylog2-server/src/main/java/org/graylog/events/context/EventDefinitionContextService.java index 7f940791aed5..287dbdcbf1ae 100644 --- a/graylog2-server/src/main/java/org/graylog/events/context/EventDefinitionContextService.java +++ b/graylog2-server/src/main/java/org/graylog/events/context/EventDefinitionContextService.java @@ -19,6 +19,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; +import com.mongodb.client.model.Filters; +import jakarta.inject.Inject; +import org.bson.conversions.Bson; import org.graylog.events.notifications.EventNotificationExecutionJob; import org.graylog.events.processor.EventDefinitionDto; import org.graylog.events.processor.EventProcessorExecutionJob; @@ -29,9 +32,6 @@ import org.graylog.scheduler.JobTriggerDto; import org.graylog.scheduler.JobTriggerStatus; import org.joda.time.DateTime; -import org.mongojack.DBQuery; - -import jakarta.inject.Inject; import java.util.Collection; import java.util.Collections; @@ -80,10 +80,10 @@ private Map> getJobTriggers(Map db; + private final MongoCollection collection; @Inject - public DBEventProcessorStateService(MongoConnection mongoConnection, - MongoJackObjectMapperProvider mapper) { - this.db = JacksonDBCollection.wrap(mongoConnection.getDatabase().getCollection(COLLECTION_NAME), - EventProcessorStateDto.class, - ObjectId.class, - mapper.get()); + public DBEventProcessorStateService(MongoCollections mongoCollections) { + collection = mongoCollections.collection(COLLECTION_NAME, EventProcessorStateDto.class); // There should only be one state document for each event processor. - db.createIndex(new BasicDBObject(FIELD_EVENT_DEFINITION_ID, 1), new BasicDBObject("unique", true)); - db.createIndex(new BasicDBObject(FIELD_MIN_PROCESSED_TIMESTAMP, 1)); - db.createIndex(new BasicDBObject(FIELD_MAX_PROCESSED_TIMESTAMP, 1)); + collection.createIndex(Indexes.ascending(FIELD_EVENT_DEFINITION_ID), new IndexOptions().unique(true)); + collection.createIndex(Indexes.ascending(FIELD_MIN_PROCESSED_TIMESTAMP)); + collection.createIndex(Indexes.ascending(FIELD_MAX_PROCESSED_TIMESTAMP)); } /** @@ -77,7 +70,7 @@ public DBEventProcessorStateService(MongoConnection mongoConnection, Optional findByEventDefinitionId(String eventDefinitionId) { checkArgument(!isNullOrEmpty(eventDefinitionId), "eventDefinitionId cannot be null or empty"); - return Optional.ofNullable(db.findOne(DBQuery.is(FIELD_EVENT_DEFINITION_ID, eventDefinitionId))); + return Optional.ofNullable(collection.find(Filters.eq(FIELD_EVENT_DEFINITION_ID, eventDefinitionId)).first()); } /** @@ -92,12 +85,12 @@ public ImmutableSet findByEventDefinitionsAndMaxTimestam checkArgument(eventDefinitionIds != null && !eventDefinitionIds.isEmpty(), "eventDefinitionIds cannot be null or empty"); checkArgument(maxTimestamp != null, "maxTimestamp cannot be null"); - final DBQuery.Query query = DBQuery.and( - DBQuery.in(FIELD_EVENT_DEFINITION_ID, eventDefinitionIds), - DBQuery.greaterThanEquals(FIELD_MAX_PROCESSED_TIMESTAMP, maxTimestamp) + final Bson query = Filters.and( + Filters.in(FIELD_EVENT_DEFINITION_ID, eventDefinitionIds), + Filters.gte(FIELD_MAX_PROCESSED_TIMESTAMP, maxTimestamp) ); - return ImmutableSet.copyOf(db.find(query).iterator()); + return ImmutableSet.copyOf(collection.find(query)); } /** @@ -133,31 +126,13 @@ public Optional setState(String eventDefinitionId, // Example: If the minProcessedTimestamp argument is newer than the value in the existing record, we don't // want to change it. The other way around for the maxProcessedTimestamp. // That's why we are using the $min and $max operations for the update query. - final DBUpdate.Builder update = DBUpdate.set(FIELD_EVENT_DEFINITION_ID, eventDefinitionId) - // Our current mongojack implementation doesn't offer $min/$max helper - .addOperation("$min", FIELD_MIN_PROCESSED_TIMESTAMP, updateValue(minProcessedTimestamp)) - .addOperation("$max", FIELD_MAX_PROCESSED_TIMESTAMP, updateValue(maxProcessedTimestamp)); - - return Optional.ofNullable(db.findAndModify( - // We have a unique index on the eventDefinitionId so this query is enough - DBQuery.is(FIELD_EVENT_DEFINITION_ID, eventDefinitionId), - null, - null, - false, - update, - true, // We want to return the updated document to the caller - true)); - } + final Bson update = Updates.combine( + Updates.set(FIELD_EVENT_DEFINITION_ID, eventDefinitionId), + Updates.min(FIELD_MIN_PROCESSED_TIMESTAMP, minProcessedTimestamp), + Updates.max(FIELD_MAX_PROCESSED_TIMESTAMP, maxProcessedTimestamp)); - /** - * Only used to create an {@link UpdateOperationValue} for - * {@link DBUpdate.Builder#addOperation(String, String, UpdateOperationValue)}. - * - * @param value the object value - * @return the update operation value - */ - private SingleUpdateOperationValue updateValue(Object value) { - return new SingleUpdateOperationValue(false, true, value); + return Optional.ofNullable(collection.findOneAndUpdate(Filters.eq(FIELD_EVENT_DEFINITION_ID, eventDefinitionId), + update, new FindOneAndUpdateOptions().upsert(true).returnDocument(ReturnDocument.AFTER))); } /** @@ -167,8 +142,6 @@ private SingleUpdateOperationValue updateValue(Object value) { * @return the number of objects that have been deleted */ public int deleteByEventDefinitionId(String id) { - return findByEventDefinitionId(id) - .map(dto -> db.removeById(new ObjectId(requireNonNull(dto.id()))).getN()) - .orElse(0); + return (int) collection.deleteOne(Filters.eq(FIELD_EVENT_DEFINITION_ID, id)).getDeletedCount(); } } diff --git a/graylog2-server/src/main/java/org/graylog/events/processor/EventDefinitionHandler.java b/graylog2-server/src/main/java/org/graylog/events/processor/EventDefinitionHandler.java index b1960311f5a1..7cda23448c10 100644 --- a/graylog2-server/src/main/java/org/graylog/events/processor/EventDefinitionHandler.java +++ b/graylog2-server/src/main/java/org/graylog/events/processor/EventDefinitionHandler.java @@ -16,6 +16,9 @@ */ package org.graylog.events.processor; +import com.mongodb.client.model.Filters; +import jakarta.inject.Inject; +import org.bson.conversions.Bson; import org.graylog.events.event.Event; import org.graylog.events.event.EventWithContext; import org.graylog.events.notifications.EventNotificationExecutionJob; @@ -28,12 +31,9 @@ import org.graylog2.database.entities.DefaultEntityScope; import org.graylog2.plugin.database.users.User; import org.joda.time.DateTime; -import org.mongojack.DBQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.inject.Inject; - import java.util.HashMap; import java.util.List; import java.util.Map; @@ -206,7 +206,7 @@ public boolean deleteImmutable(String eventDefinitionId) { private boolean doDelete(String eventDefinitionId, Supplier deleteSupplier) { final Optional optionalEventDefinition = eventDefinitionService.get(eventDefinitionId); - if (!optionalEventDefinition.isPresent()) { + if (optionalEventDefinition.isEmpty()) { return false; } @@ -387,9 +387,9 @@ private void deleteJobDefinitionAndTrigger(JobDefinitionDto jobDefinition, Event } private void deleteNotificationJobTriggers(EventDefinitionDto eventDefinition) { - final DBQuery.Query query = DBQuery.and( - DBQuery.is("data.type", EventNotificationExecutionJob.TYPE_NAME), - DBQuery.is("data.event_dto.event_definition_id", eventDefinition.id())); + final Bson query = Filters.and( + Filters.eq("data.type", EventNotificationExecutionJob.TYPE_NAME), + Filters.eq("data.event_dto.event_definition_id", eventDefinition.id())); final int numberOfDeletedTriggers = jobTriggerService.deleteByQuery(query); @@ -427,7 +427,7 @@ private void updateJobTrigger(EventDefinitionDto eventDefinition, JobDefinitionDto oldJobDefinition, EventProcessorSchedulerConfig schedulerConfig) { final Optional optionalOldJobTrigger = getJobTrigger(jobDefinition); - if (!optionalOldJobTrigger.isPresent()) { + if (optionalOldJobTrigger.isEmpty()) { // Nothing to do if there are no job triggers to update return; } @@ -460,7 +460,7 @@ private void updateJobTrigger(EventDefinitionDto eventDefinition, private void deleteJobTrigger(JobDefinitionDto jobDefinition, EventDefinitionDto eventDefinition) { final Optional optionalJobTrigger = getJobTrigger(jobDefinition); - if (!optionalJobTrigger.isPresent()) { + if (optionalJobTrigger.isEmpty()) { return; } diff --git a/graylog2-server/src/main/java/org/graylog/events/processor/EventProcessorStateDto.java b/graylog2-server/src/main/java/org/graylog/events/processor/EventProcessorStateDto.java index 64f598a90226..b608a83467f4 100644 --- a/graylog2-server/src/main/java/org/graylog/events/processor/EventProcessorStateDto.java +++ b/graylog2-server/src/main/java/org/graylog/events/processor/EventProcessorStateDto.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.auto.value.AutoValue; +import org.graylog2.database.MongoEntity; import org.joda.time.DateTime; import org.mongojack.Id; import org.mongojack.ObjectId; @@ -28,7 +29,7 @@ @AutoValue @JsonDeserialize(builder = EventProcessorStateDto.Builder.class) -public abstract class EventProcessorStateDto { +public abstract class EventProcessorStateDto implements MongoEntity { private static final String FIELD_ID = "id"; static final String FIELD_EVENT_DEFINITION_ID = "event_definition_id"; static final String FIELD_MIN_PROCESSED_TIMESTAMP = "min_processed_timestamp"; @@ -78,4 +79,4 @@ public static Builder create() { public abstract EventProcessorStateDto build(); } -} \ No newline at end of file +} diff --git a/graylog2-server/src/main/java/org/graylog/integrations/aws/cloudwatch/FlowLogMessage.java b/graylog2-server/src/main/java/org/graylog/integrations/aws/cloudwatch/FlowLogMessage.java index dc763c2510a7..e7183e7c3f65 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/aws/cloudwatch/FlowLogMessage.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/aws/cloudwatch/FlowLogMessage.java @@ -21,8 +21,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; - public class FlowLogMessage { private static final Logger LOG = LoggerFactory.getLogger(FlowLogMessage.class); @@ -74,13 +72,11 @@ private FlowLogMessage(DateTime timestamp, this.logStatus = logStatus; } - @Nullable public static FlowLogMessage fromLogEvent(final KinesisLogEntry logEvent) { final String[] parts = logEvent.message().split(" "); if (parts.length != 14) { - LOG.warn("Received FlowLog message with not exactly 14 fields. Skipping. Message was: [{}]", logEvent.message()); - return null; + throw new RuntimeException("Received FlowLog message with not exactly 14 fields. Skipping. Message was: [%s]".formatted(logEvent.message())); } return new FlowLogMessage( diff --git a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AWSCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AWSCodec.java index 652a7764fda1..424bccb8c0d8 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AWSCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AWSCodec.java @@ -17,6 +17,7 @@ package org.graylog.integrations.aws.codecs; import com.google.inject.assistedinject.Assisted; +import jakarta.inject.Inject; import org.graylog.integrations.aws.AWSMessageType; import org.graylog2.plugin.Message; import org.graylog2.plugin.configuration.Configuration; @@ -28,17 +29,15 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.Codec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.regions.Region; import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import jakarta.inject.Inject; - import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class AWSCodec extends AbstractCodec { @@ -64,28 +63,21 @@ public AWSCodec(@Assisted Configuration configuration, this.availableCodecs = availableCodecs; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { // Load the codec by message type. final AWSMessageType awsMessageType = AWSMessageType.valueOf(configuration.getString(CK_AWS_MESSAGE_TYPE)); final Codec.Factory codecFactory = this.availableCodecs.get(awsMessageType.getCodecName()); if (codecFactory == null) { - LOG.error("A codec with name [{}] could not be found.", awsMessageType.getCodecName()); - return null; + throw InputProcessingException.create("A codec with name [%s] could not be found.".formatted(awsMessageType.getCodecName()), + rawMessage, new String(rawMessage.getPayload(), charset)); } final Codec codec = codecFactory.create(configuration); // Parse the message with the specified codec. - final Message message = codec.decode(new RawMessage(rawMessage.getPayload())); - if (message == null) { - LOG.error("Failed to decode message for codec [{}].", codec.getName()); - return null; - } - - return message; + return codec.decodeSafe(new RawMessage(rawMessage.getPayload())); } @Override diff --git a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AbstractKinesisCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AbstractKinesisCodec.java index 9bae979dee59..97e0be258d58 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AbstractKinesisCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/AbstractKinesisCodec.java @@ -22,13 +22,14 @@ import org.graylog2.plugin.configuration.Configuration; import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; +import java.util.Optional; public abstract class AbstractKinesisCodec extends AbstractCodec { @@ -46,27 +47,24 @@ public abstract class AbstractKinesisCodec extends AbstractCodec { this.objectMapper = objectMapper; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { try { final KinesisLogEntry entry = objectMapper.readValue(rawMessage.getPayload(), KinesisLogEntry.class); try { return decodeLogData(entry); } catch (Exception e) { - LOG.error("Couldn't decode log event <{}>", entry); - - // Message will be dropped when returning null - return null; + throw InputProcessingException.create("Couldn't decode log event <%s>".formatted(entry), + e, rawMessage, new String(rawMessage.getPayload(), charset)); } - } catch (IOException e) { - throw new RuntimeException("Couldn't deserialize log data", e); + } catch (Exception e) { + throw InputProcessingException.create("Couldn't deserialize log data", + e, rawMessage, new String(rawMessage.getPayload(), charset)); } } - @Nullable - protected abstract Message decodeLogData(@Nonnull final KinesisLogEntry event); + protected abstract Optional decodeLogData(@Nonnull final KinesisLogEntry event); @Nonnull @Override diff --git a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisCloudWatchFlowLogCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisCloudWatchFlowLogCodec.java index 74c73e4f9a06..28f2be0d7846 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisCloudWatchFlowLogCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisCloudWatchFlowLogCodec.java @@ -33,9 +33,9 @@ import org.joda.time.Seconds; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; public class KinesisCloudWatchFlowLogCodec extends AbstractKinesisCodec { public static final String NAME = "FlowLog"; @@ -67,16 +67,11 @@ public KinesisCloudWatchFlowLogCodec(@Assisted Configuration configuration, Obje this.noFlowLogPrefix = configuration.getBoolean(AWSCodec.CK_FLOW_LOG_PREFIX, AWSCodec.FLOW_LOG_PREFIX_DEFAULT); } - @Nullable @Override - public Message decodeLogData(@Nonnull final KinesisLogEntry logEvent) { + public Optional decodeLogData(@Nonnull final KinesisLogEntry logEvent) { try { final FlowLogMessage flowLogMessage = FlowLogMessage.fromLogEvent(logEvent); - if (flowLogMessage == null) { - return null; - } - final String source = configuration.getString(KinesisCloudWatchFlowLogCodec.Config.CK_OVERRIDE_SOURCE, SOURCE); final Message result = messageFactory.createMessage( buildSummary(flowLogMessage), @@ -88,7 +83,7 @@ public Message decodeLogData(@Nonnull final KinesisLogEntry logEvent) { result.addField(FIELD_LOG_STREAM, logEvent.logStream()); result.addField(SOURCE_GROUP_IDENTIFIER, true); - return result; + return Optional.of(result); } catch (Exception e) { throw new RuntimeException("Could not deserialize AWS FlowLog record.", e); } diff --git a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisRawLogCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisRawLogCodec.java index 943ed4ddc641..fe78cf276b2a 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisRawLogCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/aws/codecs/KinesisRawLogCodec.java @@ -29,7 +29,7 @@ import org.graylog2.plugin.inputs.codecs.Codec; import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import java.util.Optional; public class KinesisRawLogCodec extends AbstractKinesisCodec { public static final String NAME = "CloudWatchRawLog"; @@ -42,9 +42,8 @@ public KinesisRawLogCodec(@Assisted Configuration configuration, ObjectMapper ob this.messageFactory = messageFactory; } - @Nullable @Override - public Message decodeLogData(@Nonnull final KinesisLogEntry logEvent) { + public Optional decodeLogData(@Nonnull final KinesisLogEntry logEvent) { try { final String source = configuration.getString(KinesisCloudWatchFlowLogCodec.Config.CK_OVERRIDE_SOURCE, SOURCE); Message result = messageFactory.createMessage( @@ -56,7 +55,7 @@ public Message decodeLogData(@Nonnull final KinesisLogEntry logEvent) { result.addField(FIELD_LOG_GROUP, logEvent.logGroup()); result.addField(FIELD_LOG_STREAM, logEvent.logStream()); - return result; + return Optional.of(result); } catch (Exception e) { throw new RuntimeException("Could not deserialize AWS FlowLog record.", e); } diff --git a/graylog2-server/src/main/java/org/graylog/integrations/aws/service/KinesisService.java b/graylog2-server/src/main/java/org/graylog/integrations/aws/service/KinesisService.java index 201dda35dc0a..6799f84fee5d 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/aws/service/KinesisService.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/aws/service/KinesisService.java @@ -23,6 +23,8 @@ import com.github.rholder.retry.RetryerBuilder; import com.github.rholder.retry.StopStrategies; import com.google.common.base.Preconditions; +import jakarta.inject.Inject; +import jakarta.ws.rs.BadRequestException; import org.apache.commons.collections.CollectionUtils; import org.graylog.integrations.aws.AWSClientBuilderUtil; import org.graylog.integrations.aws.AWSLogMessage; @@ -69,10 +71,6 @@ import software.amazon.awssdk.services.kinesis.model.StreamDescription; import software.amazon.awssdk.services.kinesis.model.StreamStatus; -import jakarta.inject.Inject; - -import jakarta.ws.rs.BadRequestException; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -359,11 +357,9 @@ private KinesisHealthCheckResponse detectAndParseMessage(String logMessage, Date throw new BadRequestException("Encoding the message to bytes failed.", e); } - final Message fullyParsedMessage = codec.decode(new RawMessage(payload)); - if (fullyParsedMessage == null) { - throw new BadRequestException(String.format(Locale.ROOT, "Message decoding failed. More information might be " + - "available by enabling Debug logging. message [%s]", logMessage)); - } + final Message fullyParsedMessage = codec.decodeSafe(new RawMessage(payload)).orElseThrow(() -> + new BadRequestException(String.format(Locale.ROOT, "Message decoding failed. More information might be " + + "available by enabling Debug logging. message [%s]", logMessage))); LOG.debug("Successfully parsed message type [{}] with codec [{}].", awsMessageType, awsMessageType.getCodecName()); diff --git a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodec.java index 675490d276de..c6c16b066742 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodec.java @@ -29,6 +29,7 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.Codec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTimeZone; import org.slf4j.Logger; @@ -37,6 +38,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.nio.charset.StandardCharsets; +import java.util.Optional; public class PaloAltoCodec implements Codec { @@ -64,9 +66,8 @@ public PaloAltoCodec(@Assisted Configuration configuration, MessageFactory messa configuration.getString(CK_TRAFFIC_TEMPLATE, PaloAltoTemplateDefaults.TRAFFIC_TEMPLATE)); } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { String s = new String(rawMessage.getPayload(), StandardCharsets.UTF_8); LOG.trace("Received raw message: {}", s); @@ -74,35 +75,33 @@ public Message decode(@Nonnull RawMessage rawMessage) { // previously existing PA inputs after updating will not have a Time Zone configured, default to UTC DateTimeZone timezone = timezoneID != null ? DateTimeZone.forID(timezoneID) : DateTimeZone.UTC; LOG.trace("Configured time zone: {}", timezone); - PaloAltoMessageBase p = parser.parse(s, timezone); - - // Return when error occurs parsing syslog header. - if (p == null) { - return null; - } - - Message message = messageFactory.createMessage(p.payload(), p.source(), p.timestamp()); - - switch (p.panType()) { - case "THREAT": - final PaloAltoTypeParser parserThreat = new PaloAltoTypeParser(templates.getThreatMessageTemplate()); - message.addFields(parserThreat.parseFields(p.fields(), timezone)); - break; - case "SYSTEM": - final PaloAltoTypeParser parserSystem = new PaloAltoTypeParser(templates.getSystemMessageTemplate()); - message.addFields(parserSystem.parseFields(p.fields(), timezone)); - break; - case "TRAFFIC": - final PaloAltoTypeParser parserTraffic = new PaloAltoTypeParser(templates.getTrafficMessageTemplate()); - message.addFields(parserTraffic.parseFields(p.fields(), timezone)); - break; - default: - LOG.error("Unsupported PAN type [{}]. Not adding any parsed fields.", p.panType()); + try { + PaloAltoMessageBase p = parser.parse(s, timezone); + Message message = messageFactory.createMessage(p.payload(), p.source(), p.timestamp()); + + switch (p.panType()) { + case "THREAT": + final PaloAltoTypeParser parserThreat = new PaloAltoTypeParser(templates.getThreatMessageTemplate()); + message.addFields(parserThreat.parseFields(p.fields(), timezone)); + break; + case "SYSTEM": + final PaloAltoTypeParser parserSystem = new PaloAltoTypeParser(templates.getSystemMessageTemplate()); + message.addFields(parserSystem.parseFields(p.fields(), timezone)); + break; + case "TRAFFIC": + final PaloAltoTypeParser parserTraffic = new PaloAltoTypeParser(templates.getTrafficMessageTemplate()); + message.addFields(parserTraffic.parseFields(p.fields(), timezone)); + break; + default: + LOG.error("Unsupported PAN type [{}]. Not adding any parsed fields.", p.panType()); + } + + LOG.trace("Successfully processed [{}] message with [{}] fields.", p.panType(), message.getFieldCount()); + + return Optional.of(message); + } catch (Exception e) { + throw InputProcessingException.create("Could not decode PaloAlto9x message.", e, rawMessage, s); } - - LOG.trace("Successfully processed [{}] message with [{}] fields.", p.panType(), message.getFieldCount()); - - return message; } @Override diff --git a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoParser.java b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoParser.java index 981c6902e887..e821bfd3d7cd 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoParser.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto/PaloAltoParser.java @@ -18,6 +18,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import jakarta.annotation.Nonnull; +import jakarta.validation.constraints.NotNull; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -30,10 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; - -import jakarta.validation.constraints.NotNull; - import java.io.IOException; import java.io.Reader; import java.io.StringReader; @@ -65,7 +63,7 @@ public class PaloAltoParser { // Used to remove extra space between month and day, so date parsing works. eg. Apr 8 01:47:32 -> Apr 8 01:47:32 private static final String DOUBLE_SPACE = "\\s{2}"; - @Nullable + @Nonnull public PaloAltoMessageBase parse(@NotNull String raw, DateTimeZone timezone) { /* The message might arrive in one of the following formats: @@ -99,8 +97,7 @@ public PaloAltoMessageBase parse(@NotNull String raw, DateTimeZone timezone) { return buildPaloAltoMessageBase(timestamp, fieldsString, source); } else { - LOG.error("Cannot parse malformed Panorama message: {}", raw); - return null; + throw new IllegalArgumentException("Cannot parse malformed Panorama message: " + raw); } } else { if (STANDARD_SYSLOG_PARSER.matcher(raw).matches()) { @@ -120,8 +117,7 @@ public PaloAltoMessageBase parse(@NotNull String raw, DateTimeZone timezone) { return buildPaloAltoMessageBase(timestamp, panData, source); } else { - LOG.error("Cannot parse malformed Syslog message: {}", raw); - return null; + throw new IllegalArgumentException("Cannot parse malformed Syslog message: " + raw); } } else if (STANDARD_SYSLOG_NO_HOST_PARSER.matcher(raw).matches()) { LOG.trace("Message is in structured syslog (with no hostname) format [{}]", raw); @@ -137,14 +133,11 @@ public PaloAltoMessageBase parse(@NotNull String raw, DateTimeZone timezone) { // No source is supplied, so use a blank one return buildPaloAltoMessageBase(timestamp, panData, ""); } else { - LOG.error("Cannot parse malformed Syslog message: {}", raw); - return null; + throw new IllegalArgumentException("Cannot parse malformed Syslog message: " + raw); } } } - - LOG.error("Cannot parse malformed PAN message [unrecognized format]: {}", raw); - return null; + throw new IllegalArgumentException("Cannot parse malformed PAN message [unrecognized format]"); } /** diff --git a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodec.java index d3e1d4cd92a8..7a26a5b5b54d 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodec.java @@ -44,6 +44,7 @@ import javax.annotation.Nullable; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.Optional; public class PaloAlto11xCodec implements Codec { private static final Logger LOG = LoggerFactory.getLogger(PaloAlto11xCodec.class); @@ -71,9 +72,8 @@ public PaloAlto11xCodec(@Assisted Configuration configuration, PaloAltoParser ra this.rawMessageParser = rawMessageParser; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { String rawMessageString = new String(rawMessage.getPayload(), StandardCharsets.UTF_8); LOG.trace("Received raw message: {}", rawMessageString); @@ -111,7 +111,7 @@ public Message decode(@Nonnull RawMessage rawMessage) { message.addField(Message.FIELD_FULL_MESSAGE, new String(rawMessage.getPayload(), StandardCharsets.UTF_8)); } LOG.trace("Successfully processed [{}] message with [{}] fields.", panType, message.getFieldCount()); - return message; + return Optional.of(message); } private String getRawMessageSource(RawMessage rawMessage) { diff --git a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodec.java index 20defbe8a67a..6a82289dc75c 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodec.java @@ -32,6 +32,7 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.Codec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTimeZone; import org.slf4j.Logger; @@ -40,6 +41,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.nio.charset.StandardCharsets; +import java.util.Optional; import static org.graylog.integrations.inputs.paloalto.PaloAltoMessageType.CONFIG; import static org.graylog.integrations.inputs.paloalto.PaloAltoMessageType.CORRELATION; @@ -78,67 +80,65 @@ public PaloAlto9xCodec(@Assisted Configuration configuration, PaloAltoParser raw this.fieldProducer = fieldProducer; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { String s = new String(rawMessage.getPayload(), StandardCharsets.UTF_8); LOG.trace("Received raw message: {}", s); - PaloAltoMessageBase p = rawMessageParser.parse(s, timezone); - // Return when error occurs parsing syslog header. - if (p == null) { - return null; - } + try { + PaloAltoMessageBase p = rawMessageParser.parse(s, timezone); + Message message = messageFactory.createMessage(p.payload(), p.source(), p.timestamp()); - Message message = messageFactory.createMessage(p.payload(), p.source(), p.timestamp()); - - switch (p.panType()) { - case "THREAT": - message.addFields(fieldProducer.parseFields(THREAT, p.fields(), timezone)); - break; - case "SYSTEM": - message.addFields(fieldProducer.parseFields(SYSTEM, p.fields(), timezone)); - break; - case "TRAFFIC": - message.addFields(fieldProducer.parseFields(TRAFFIC, p.fields(), timezone)); - break; - case "CONFIG": - message.addFields(fieldProducer.parseFields(CONFIG, p.fields(), timezone)); - break; - case "HIP-MATCH": - case "HIPMATCH": - message.addFields(fieldProducer.parseFields(HIP, p.fields(), timezone)); - break; - case "CORRELATION": - message.addFields(fieldProducer.parseFields(CORRELATION, p.fields(), timezone)); - break; - case "GLOBALPROTECT": - // For PAN v9.1.3 and later, Global Protect has type in the expected position - message.addFields(fieldProducer.parseFields(GLOBAL_PROTECT_9_1_3, p.fields(), timezone)); - break; - case "USERID": - message.addFields(fieldProducer.parseFields(USERID, p.fields(), timezone)); - break; - default: - //For PAN v9.1.2 and earlier, Global Protect has type in position 5 rather than position 3 - if (p.fields().get(5).equals("GLOBALPROTECT")) { - message.addFields(fieldProducer.parseFields(GLOBAL_PROTECT_PRE_9_1_3, p.fields(), timezone)); + switch (p.panType()) { + case "THREAT": + message.addFields(fieldProducer.parseFields(THREAT, p.fields(), timezone)); break; - } else { - LOG.info("Received log for unsupported PAN type [{}]. Will not parse.", p.panType()); - } - } - - message.addField(EventFields.EVENT_SOURCE_PRODUCT, "PAN"); - - // Store full message if configured. - if (configuration.getBoolean(CK_STORE_FULL_MESSAGE)) { - message.addField(Message.FIELD_FULL_MESSAGE, new String(rawMessage.getPayload(), StandardCharsets.UTF_8)); + case "SYSTEM": + message.addFields(fieldProducer.parseFields(SYSTEM, p.fields(), timezone)); + break; + case "TRAFFIC": + message.addFields(fieldProducer.parseFields(TRAFFIC, p.fields(), timezone)); + break; + case "CONFIG": + message.addFields(fieldProducer.parseFields(CONFIG, p.fields(), timezone)); + break; + case "HIP-MATCH": + case "HIPMATCH": + message.addFields(fieldProducer.parseFields(HIP, p.fields(), timezone)); + break; + case "CORRELATION": + message.addFields(fieldProducer.parseFields(CORRELATION, p.fields(), timezone)); + break; + case "GLOBALPROTECT": + // For PAN v9.1.3 and later, Global Protect has type in the expected position + message.addFields(fieldProducer.parseFields(GLOBAL_PROTECT_9_1_3, p.fields(), timezone)); + break; + case "USERID": + message.addFields(fieldProducer.parseFields(USERID, p.fields(), timezone)); + break; + default: + //For PAN v9.1.2 and earlier, Global Protect has type in position 5 rather than position 3 + if (p.fields().get(5).equals("GLOBALPROTECT")) { + message.addFields(fieldProducer.parseFields(GLOBAL_PROTECT_PRE_9_1_3, p.fields(), timezone)); + break; + } else { + LOG.info("Received log for unsupported PAN type [{}]. Will not parse.", p.panType()); + } + } + + message.addField(EventFields.EVENT_SOURCE_PRODUCT, "PAN"); + + // Store full message if configured. + if (configuration.getBoolean(CK_STORE_FULL_MESSAGE)) { + message.addField(Message.FIELD_FULL_MESSAGE, new String(rawMessage.getPayload(), StandardCharsets.UTF_8)); + } + + LOG.trace("Successfully processed [{}] message with [{}] fields.", p.panType(), message.getFieldCount()); + + return Optional.of(message); + } catch (Exception e) { + throw InputProcessingException.create("Could not decode PaloAlto9x message.", e, rawMessage, s); } - - LOG.trace("Successfully processed [{}] message with [{}] fields.", p.panType(), message.getFieldCount()); - - return message; } @Override diff --git a/graylog2-server/src/main/java/org/graylog/integrations/ipfix/codecs/IpfixCodec.java b/graylog2-server/src/main/java/org/graylog/integrations/ipfix/codecs/IpfixCodec.java index 4be4d89d2c0d..58dedc5c242d 100644 --- a/graylog2-server/src/main/java/org/graylog/integrations/ipfix/codecs/IpfixCodec.java +++ b/graylog2-server/src/main/java/org/graylog/integrations/ipfix/codecs/IpfixCodec.java @@ -21,7 +21,6 @@ import com.google.common.io.Resources; import com.google.inject.assistedinject.Assisted; import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import io.netty.buffer.Unpooled; import jakarta.inject.Inject; import org.graylog.integrations.ipfix.Flow; @@ -43,6 +42,7 @@ import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; import org.graylog2.plugin.inputs.codecs.MultiMessageCodec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.inputs.transports.NettyTransport; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTime; @@ -69,6 +69,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -198,7 +199,7 @@ public Collection decodeMessages(@Nonnull RawMessage rawMessage) { .toMap(Tuple2::v1, Tuple2::v2); return rawIpfix.getDataSetsList().stream() - .map(dataSet -> { + .flatMap(dataSet -> { final int templateId = dataSet.getTemplateId(); final ZonedDateTime flowExportTimestamp = ZonedDateTime.ofInstant(Instant.ofEpochSecond(dataSet.getTimestampEpochSeconds()), ZoneOffset.UTC); final TemplateRecord templateRecord = templateRecordMap.get(templateId); @@ -210,11 +211,9 @@ public Collection decodeMessages(@Nonnull RawMessage rawMessage) { return flows.stream() .map(flow -> formatFlow(flowExportTimestamp, sender, flow)); }) - .flatMap(messageStream -> messageStream) .collect(Collectors.toList()); - } catch (InvalidProtocolBufferException e) { - LOG.error("Unable to parse ipfix journal message", e); - return Collections.emptyList(); + } catch (Exception e) { + throw InputProcessingException.create("Unable to parse ipfix journal message", rawMessage); } } @@ -227,9 +226,8 @@ private Message formatFlow(ZonedDateTime flowExportTimestamp, InetSocketAddress return message; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { throw new UnsupportedOperationException("MultiMessageCodec " + getClass() + " does not support decode()"); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java b/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java index b745d8557ce6..87e589ee47ce 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java @@ -31,6 +31,7 @@ import org.graylog2.plugin.inputs.annotations.ConfigClass; import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTime; import org.slf4j.Logger; @@ -38,11 +39,11 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -66,24 +67,25 @@ public Beats2Codec(@Assisted Configuration configuration, ObjectMapper objectMap this.messageFactory = messageFactory; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { - final byte[] payload = rawMessage.getPayload(); - final JsonNode event; + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { try { + final byte[] payload = rawMessage.getPayload(); + final JsonNode event; event = objectMapper.readTree(payload); if (event == null || event.isMissingNode()) { - throw new IOException("null result"); + throw InputProcessingException.create("Decoded message is null or empty!", rawMessage); } - } catch (IOException e) { - LOG.error("Couldn't decode raw message {}", rawMessage); - return null; + return Optional.of(parseEvent(event)); + } catch (InputProcessingException e) { + throw e; + } catch (Exception e) { + throw InputProcessingException.create("Couldn't decode beats 2 message", + e, rawMessage, new String(rawMessage.getPayload(), charset)); } - - return parseEvent(event); } + @Nonnull private Message parseEvent(JsonNode event) { final String beatsType = event.path("@metadata").path("beat").asText("beat"); final String rootPath = noBeatsPrefix ? "" : beatsType; diff --git a/graylog2-server/src/main/java/org/graylog/plugins/beats/BeatsCodec.java b/graylog2-server/src/main/java/org/graylog/plugins/beats/BeatsCodec.java index f004807445c1..996010ee46c2 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/beats/BeatsCodec.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/beats/BeatsCodec.java @@ -28,16 +28,16 @@ import org.graylog2.plugin.inputs.annotations.ConfigClass; import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -56,22 +56,18 @@ public BeatsCodec(@Assisted Configuration configuration, ObjectMapper objectMapp this.messageFactory = messageFactory; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { - final byte[] payload = rawMessage.getPayload(); - final Map event; + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { try { - event = objectMapper.readValue(payload, TypeReferences.MAP_STRING_OBJECT); - } catch (IOException e) { - LOG.error("Couldn't decode raw message {}", rawMessage); - return null; + final byte[] payload = rawMessage.getPayload(); + return Optional.of(parseEvent(objectMapper.readValue(payload, TypeReferences.MAP_STRING_OBJECT))); + } catch (Exception e) { + throw InputProcessingException.create("Couldn't decode beats message", + e, rawMessage, new String(rawMessage.getPayload(), charset)); } - - return parseEvent(event); } - @Nullable + @Nonnull private Message parseEvent(Map event) { @SuppressWarnings("unchecked") final Map metadata = (HashMap) event.remove("@metadata"); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/cef/codec/CEFCodec.java b/graylog2-server/src/main/java/org/graylog/plugins/cef/codec/CEFCodec.java index 0d09a29a9be7..b172c252a7d4 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/cef/codec/CEFCodec.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/cef/codec/CEFCodec.java @@ -37,6 +37,7 @@ import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.Codec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.graylog2.shared.SuppressForbidden; import org.joda.time.DateTime; @@ -50,6 +51,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -94,29 +96,32 @@ public CEFCodec(@Assisted Configuration configuration, MessageFactory messageFac this.useFullNames = configuration.getBoolean(CK_USE_FULL_NAMES); } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { final String s = new String(rawMessage.getPayload(), charset); - final Matcher matcher = SYSLOG_PREFIX.matcher(s); - - if (matcher.find()) { - final String priString = matcher.group("pri"); - final Integer pri = Ints.tryParse(priString); - final Map syslogFields = new HashMap<>(); - if (pri != null) { - final int facility = SyslogUtils.facilityFromPriority(pri); - syslogFields.put("level", SyslogUtils.levelFromPriority(pri)); - syslogFields.put("facility", SyslogUtils.facilityToString(facility)); + try { + final Matcher matcher = SYSLOG_PREFIX.matcher(s); + + if (matcher.find()) { + final String priString = matcher.group("pri"); + final Integer pri = Ints.tryParse(priString); + final Map syslogFields = new HashMap<>(); + if (pri != null) { + final int facility = SyslogUtils.facilityFromPriority(pri); + syslogFields.put("level", SyslogUtils.levelFromPriority(pri)); + syslogFields.put("facility", SyslogUtils.facilityToString(facility)); + } + + final String msg = matcher.group("msg"); + final Message message = decodeCEF(rawMessage, msg); + message.addFields(syslogFields); + + return Optional.of(message); + } else { + return Optional.of(decodeCEF(rawMessage, s)); } - - final String msg = matcher.group("msg"); - final Message message = decodeCEF(rawMessage, msg); - message.addFields(syslogFields); - - return message; - } else { - return decodeCEF(rawMessage, s); + } catch (Exception e) { + throw InputProcessingException.create("Could not decode CEF message.", e, rawMessage, s); } } @@ -125,28 +130,25 @@ public String getName() { return NAME; } + @Nonnull protected Message decodeCEF(@Nonnull RawMessage rawMessage, String s) { - try { - final MappedMessage cef = new MappedMessage(parser.parse(s, timezone.toTimeZone(), locale), useFullNames); + final MappedMessage cef = new MappedMessage(parser.parse(s, timezone.toTimeZone(), locale), useFullNames); - // Build standard message. - Message result = messageFactory.createMessage(buildMessageSummary(cef), decideSource(cef, rawMessage), new DateTime(cef.timestamp())); + // Build standard message. + Message result = messageFactory.createMessage(buildMessageSummary(cef), decideSource(cef, rawMessage), new DateTime(cef.timestamp())); - // Add all extensions. - result.addFields(cef.mappedExtensions()); + // Add all extensions. + result.addFields(cef.mappedExtensions()); - // Add standard CEF fields. - result.addField("device_vendor", cef.deviceVendor()); - result.addField("device_product", cef.deviceProduct()); - result.addField("device_version", cef.deviceVersion()); - result.addField("event_class_id", cef.deviceEventClassId()); - result.addField("name", cef.name()); - result.addField("severity", cef.severity()); + // Add standard CEF fields. + result.addField("device_vendor", cef.deviceVendor()); + result.addField("device_product", cef.deviceProduct()); + result.addField("device_version", cef.deviceVersion()); + result.addField("event_class_id", cef.deviceEventClassId()); + result.addField("name", cef.name()); + result.addField("severity", cef.severity()); - return result; - } catch (Exception e) { - throw new RuntimeException("Could not decode CEF message.", e); - } + return result; } protected String buildMessageSummary(com.github.jcustenborder.cef.Message cef) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/netflow/codecs/NetFlowCodec.java b/graylog2-server/src/main/java/org/graylog/plugins/netflow/codecs/NetFlowCodec.java index efa0d7acddf3..326f36ffe17d 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/netflow/codecs/NetFlowCodec.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/netflow/codecs/NetFlowCodec.java @@ -47,6 +47,7 @@ import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; import org.graylog2.plugin.inputs.codecs.MultiMessageCodec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.inputs.transports.NettyTransport; import org.graylog2.plugin.journal.RawMessage; import org.graylog2.shared.utilities.ExceptionUtils; @@ -62,6 +63,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @Codec(name = "netflow", displayName = "NetFlow") @@ -105,9 +107,8 @@ public CodecAggregator getAggregator() { return netflowV9CodecAggregator; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { throw new UnsupportedOperationException("MultiMessageCodec " + getClass() + " does not support decode()"); } @@ -120,9 +121,8 @@ public Collection decodeMessages(@Nonnull RawMessage rawMessage) { final byte[] payload = rawMessage.getPayload(); if (payload.length < 3) { - LOG.debug("NetFlow message (source: {}) doesn't even fit the NetFlow version (size: {} bytes)", - sender, payload.length); - return null; + throw InputProcessingException.create("NetFlow message (source: %s) doesn't even fit the NetFlow version (size: %s bytes)" + .formatted(sender, payload.length), rawMessage); } final ByteBuf buffer = Unpooled.wrappedBuffer(payload); @@ -140,18 +140,25 @@ public Collection decodeMessages(@Nonnull RawMessage rawMessage) { final List sourceNodes = rawMessage.getSourceNodes(); final RawMessage.SourceNode sourceNode = sourceNodes.isEmpty() ? null : sourceNodes.get(sourceNodes.size() - 1); final String inputId = sourceNode == null ? "" : sourceNode.inputId; - LOG.warn("Unsupported NetFlow packet on input {} (source: {})", inputId, sender); - return null; + throw InputProcessingException.create("Unsupported NetFlow packet on input %s (source: %s)" + .formatted(inputId, sender), rawMessage); + } } catch (FlowException e) { - LOG.error("Error parsing NetFlow packet <{}> received from <{}>", rawMessage.getId(), rawMessage.getRemoteAddress(), e); if (LOG.isDebugEnabled()) { LOG.debug("NetFlow packet hexdump:\n{}", ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(rawMessage.getPayload()))); } - return null; + throw InputProcessingException.create( + "Error parsing NetFlow packet <%s> received from <%s>".formatted(rawMessage.getId(), rawMessage.getRemoteAddress()), + e, + rawMessage, + ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(rawMessage.getPayload()))); } catch (InvalidProtocolBufferException e) { - LOG.error("Invalid NetFlowV9 entry found, cannot parse the messages", ExceptionUtils.getRootCause(e)); - return null; + throw InputProcessingException.create( + "Invalid NetFlowV9 entry found, cannot parse the messages", + ExceptionUtils.getRootCause(e), + rawMessage, + ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(rawMessage.getPayload()))); } } @@ -181,9 +188,8 @@ List decodeV9Packets(ByteBuf buffer) throws InvalidProtocolBuff templateMap.put(templateId, netFlowV9Template); }); final NetFlowV9OptionTemplate[] optionTemplate = {null}; - rawNetflowV9.getOptionTemplateMap().forEach((templateId, byteString) -> { - optionTemplate[0] = NetFlowV9Parser.parseOptionTemplate(Unpooled.wrappedBuffer(byteString.toByteArray()), typeRegistry); - }); + rawNetflowV9.getOptionTemplateMap().forEach((templateId, byteString) -> + optionTemplate[0] = NetFlowV9Parser.parseOptionTemplate(Unpooled.wrappedBuffer(byteString.toByteArray()), typeRegistry)); return rawNetflowV9.getPacketsList().stream() .map(bytes -> Unpooled.wrappedBuffer(bytes.toByteArray())) diff --git a/graylog2-server/src/main/java/org/graylog/scheduler/JobSchedule.java b/graylog2-server/src/main/java/org/graylog/scheduler/JobSchedule.java index 22595138be45..fd3390a59e58 100644 --- a/graylog2-server/src/main/java/org/graylog/scheduler/JobSchedule.java +++ b/graylog2-server/src/main/java/org/graylog/scheduler/JobSchedule.java @@ -21,10 +21,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.graylog.scheduler.clock.JobSchedulerClock; import org.joda.time.DateTime; -import org.mongojack.DBQuery; -import org.mongojack.DBUpdate; -import java.util.Map; import java.util.Optional; @JsonTypeInfo( @@ -58,19 +55,6 @@ public interface JobSchedule { @JsonIgnore Optional calculateNextTime(DateTime lastExecutionTime, DateTime lastNextTime, JobSchedulerClock clock); - /** - * Returns a map with the schedule data. This can be used to update a MongoDB document with schedule - * data. (see {@link org.mongojack.JacksonDBCollection#update(DBQuery.Query, DBUpdate.Builder) JacksonDBCollection#update()}) - * - * @param fieldPrefix the field prefix to use for the map key - * @return filled optional with a map, empty optional if there is no update data - * @deprecated This method won't be called by the persistence layer anymore. - */ - @Deprecated - default Optional> toDBUpdate(String fieldPrefix) { - return Optional.empty(); - } - interface Builder { @JsonProperty(TYPE_FIELD) SELF type(String type); diff --git a/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java b/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java index 03f1f5655976..5992158e8cda 100644 --- a/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java +++ b/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; @Command(name = "decode", description = "Decodes messages from the journal") public class JournalDecode extends AbstractJournalCommand { @@ -109,14 +110,14 @@ protected void runCommand() { } final Codec codec = codecFactory.get(raw.getCodecName()).create(raw.getCodecConfig()); - final Message message = codec.decode(raw); - if (message == null) { + final Optional message = codec.decodeSafe(raw); + if (message.isEmpty()) { System.err.println(MessageFormatter.format( "Could not use codec {} to decode raw message id {} at offset {}", new Object[]{raw.getCodecName(), raw.getId(), entry.getOffset()})); } else { - message.setMessageQueueId(raw.getMessageQueueId()); - message.setSequenceNr(raw.getSequenceNr()); + message.get().setMessageQueueId(raw.getMessageQueueId()); + message.get().setSequenceNr(raw.getSequenceNr()); } final ResolvableInetSocketAddress remoteAddress = raw.getRemoteAddress(); @@ -129,9 +130,9 @@ protected void runCommand() { .append(" at offset ").append(raw.getMessageQueueId()).append('\n') .append(" seq number ").append(raw.getSequenceNr()).append('\n') .append(" received from remote address ").append(remote).append('\n') - .append(" (source field: ").append(message == null ? "unparsed" : message.getSource()).append(')').append('\n'); - if (message != null) { - sb.append(" contains ").append(message.getFieldNames().size()).append(" fields."); + .append(" (source field: ").append(message.isEmpty() ? "unparsed" : message.get().getSource()).append(')').append('\n'); + if (message.isPresent()) { + sb.append(" contains ").append(message.get().getFieldNames().size()).append(" fields."); } else { sb.append("The message could not be parse by the given codec."); } diff --git a/graylog2-server/src/main/java/org/graylog2/events/ClusterEvent.java b/graylog2-server/src/main/java/org/graylog2/events/ClusterEvent.java index cb6ba1ff111e..6abd84832bd0 100644 --- a/graylog2-server/src/main/java/org/graylog2/events/ClusterEvent.java +++ b/graylog2-server/src/main/java/org/graylog2/events/ClusterEvent.java @@ -20,24 +20,23 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import org.graylog.autovalue.WithBeanGetter; +import org.graylog2.database.MongoEntity; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.mongojack.Id; import org.mongojack.ObjectId; import javax.annotation.Nullable; - -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; - import java.util.Collections; import java.util.Set; @JsonAutoDetect @AutoValue @WithBeanGetter -public abstract class ClusterEvent { +public abstract class ClusterEvent implements MongoEntity { @Id @ObjectId @Nullable @@ -64,7 +63,7 @@ public abstract class ClusterEvent { @JsonCreator - public static ClusterEvent create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + public static ClusterEvent create(@Id @ObjectId @JsonProperty("id") @Nullable String id, @JsonProperty("timestamp") long timestamp, @JsonProperty("producer") @Nullable String producer, @JsonProperty("consumers") @Nullable Set consumers, diff --git a/graylog2-server/src/main/java/org/graylog2/events/ClusterEventCleanupPeriodical.java b/graylog2-server/src/main/java/org/graylog2/events/ClusterEventCleanupPeriodical.java index 14e1fa4c5448..fef266c8d132 100644 --- a/graylog2-server/src/main/java/org/graylog2/events/ClusterEventCleanupPeriodical.java +++ b/graylog2-server/src/main/java/org/graylog2/events/ClusterEventCleanupPeriodical.java @@ -19,23 +19,18 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; import com.mongodb.WriteConcern; -import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; -import org.graylog2.database.MongoConnection; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import jakarta.inject.Inject; +import org.graylog2.database.MongoCollections; import org.graylog2.plugin.periodical.Periodical; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.mongojack.DBQuery; -import org.mongojack.JacksonDBCollection; -import org.mongojack.WriteResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.inject.Inject; - import java.util.concurrent.TimeUnit; -import static com.google.common.base.Preconditions.checkNotNull; - public class ClusterEventCleanupPeriodical extends Periodical { private static final Logger LOG = LoggerFactory.getLogger(ClusterEventCleanupPeriodical.class); private static final String COLLECTION_NAME = ClusterEventPeriodical.COLLECTION_NAME; @@ -43,19 +38,12 @@ public class ClusterEventCleanupPeriodical extends Periodical { @VisibleForTesting static final long DEFAULT_MAX_EVENT_AGE = TimeUnit.DAYS.toMillis(1L); - private final JacksonDBCollection dbCollection; - private final long maxEventAge; + private final MongoCollection collection; @Inject - public ClusterEventCleanupPeriodical(final MongoJackObjectMapperProvider mapperProvider, - final MongoConnection mongoConnection) { - this(JacksonDBCollection.wrap(mongoConnection.getDatabase().getCollection(COLLECTION_NAME), - ClusterEvent.class, String.class, mapperProvider.get()), DEFAULT_MAX_EVENT_AGE); - } - - ClusterEventCleanupPeriodical(final JacksonDBCollection dbCollection, final long maxEventAge) { - this.dbCollection = checkNotNull(dbCollection); - this.maxEventAge = maxEventAge; + public ClusterEventCleanupPeriodical(MongoCollections mongoCollections) { + this.collection = mongoCollections.collection(COLLECTION_NAME, ClusterEvent.class) + .withWriteConcern(WriteConcern.JOURNALED); } @Override @@ -103,11 +91,10 @@ public void doRun() { try { LOG.debug("Removing stale events from MongoDB collection \"{}\"", COLLECTION_NAME); - final long timestamp = DateTime.now(DateTimeZone.UTC).getMillis() - maxEventAge; - final DBQuery.Query query = DBQuery.lessThan("timestamp", timestamp); - final WriteResult writeResult = dbCollection.remove(query, WriteConcern.JOURNALED); + final long timestamp = DateTime.now(DateTimeZone.UTC).getMillis() - DEFAULT_MAX_EVENT_AGE; + final var deleted = collection.deleteMany(Filters.lt("timestamp", timestamp)).getDeletedCount(); - LOG.debug("Removed {} stale events from \"{}\"", writeResult.getN(), COLLECTION_NAME); + LOG.debug("Removed {} stale events from \"{}\"", deleted, COLLECTION_NAME); } catch (Exception e) { LOG.warn("Error while removing stale cluster events from MongoDB", e); } diff --git a/graylog2-server/src/main/java/org/graylog2/events/ClusterEventPeriodical.java b/graylog2-server/src/main/java/org/graylog2/events/ClusterEventPeriodical.java index 6dbc39fe4c81..513c698d2624 100644 --- a/graylog2-server/src/main/java/org/graylog2/events/ClusterEventPeriodical.java +++ b/graylog2-server/src/main/java/org/graylog2/events/ClusterEventPeriodical.java @@ -21,41 +21,36 @@ import com.google.common.eventbus.DeadEvent; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; -import com.mongodb.BasicDBList; -import com.mongodb.BasicDBObject; -import com.mongodb.DB; -import com.mongodb.DBCollection; -import com.mongodb.DBObject; import com.mongodb.MongoException; import com.mongodb.WriteConcern; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Indexes; +import com.mongodb.client.model.Sorts; +import com.mongodb.client.model.Updates; import jakarta.inject.Inject; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoCollections; import org.graylog2.database.MongoConnection; +import org.graylog2.database.utils.MongoUtils; import org.graylog2.plugin.periodical.Periodical; import org.graylog2.plugin.system.NodeId; import org.graylog2.security.RestrictedChainingClassLoader; -import org.graylog2.security.SafeClasses; import org.graylog2.security.UnsafeClassLoadingAttemptException; -import org.graylog2.shared.plugins.ChainingClassLoader; import org.graylog2.shared.utilities.AutoValueUtils; -import org.mongojack.DBCursor; -import org.mongojack.DBSort; -import org.mongojack.DBUpdate; -import org.mongojack.JacksonDBCollection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; -import static com.google.common.base.Preconditions.checkNotNull; - public class ClusterEventPeriodical extends Periodical { private static final Logger LOG = LoggerFactory.getLogger(ClusterEventPeriodical.class); @VisibleForTesting static final String COLLECTION_NAME = "cluster_events"; - private final JacksonDBCollection dbCollection; + private final MongoCollection collection; private final NodeId nodeId; private final ObjectMapper objectMapper; private final EventBus serverEventBus; @@ -68,52 +63,28 @@ public ClusterEventPeriodical(final MongoJackObjectMapperProvider mapperProvider final RestrictedChainingClassLoader chainingClassLoader, final EventBus serverEventBus, final ClusterEventBus clusterEventBus) { - this(JacksonDBCollection.wrap(prepareCollection(mongoConnection), ClusterEvent.class, String.class, mapperProvider.get()), - nodeId, mapperProvider.get(), chainingClassLoader, serverEventBus, clusterEventBus); - } - - @Deprecated - public ClusterEventPeriodical(final MongoJackObjectMapperProvider mapperProvider, - final MongoConnection mongoConnection, - final NodeId nodeId, - final ChainingClassLoader chainingClassLoader, - final EventBus serverEventBus, - final ClusterEventBus clusterEventBus) { - this(JacksonDBCollection.wrap(prepareCollection(mongoConnection), ClusterEvent.class, String.class, - mapperProvider.get()), nodeId, mapperProvider.get(), - new RestrictedChainingClassLoader(chainingClassLoader, SafeClasses.allGraylogInternal()), - serverEventBus, clusterEventBus); - } - - private ClusterEventPeriodical(final JacksonDBCollection dbCollection, - final NodeId nodeId, - final ObjectMapper objectMapper, - final RestrictedChainingClassLoader chainingClassLoader, - final EventBus serverEventBus, - final ClusterEventBus clusterEventBus) { - this.nodeId = checkNotNull(nodeId); - this.dbCollection = checkNotNull(dbCollection); - this.objectMapper = checkNotNull(objectMapper); + this.nodeId = nodeId; + this.objectMapper = mapperProvider.get(); this.chainingClassLoader = chainingClassLoader; - this.serverEventBus = checkNotNull(serverEventBus); + this.serverEventBus = serverEventBus; + this.collection = prepareCollection(mongoConnection, mapperProvider); - checkNotNull(clusterEventBus).registerClusterEventSubscriber(this); + clusterEventBus.registerClusterEventSubscriber(this); } @VisibleForTesting - static DBCollection prepareCollection(final MongoConnection mongoConnection) { - final DB db = mongoConnection.getDatabase(); - - DBCollection coll = db.getCollection(COLLECTION_NAME); + static MongoCollection prepareCollection(final MongoConnection mongoConnection, + final MongoJackObjectMapperProvider mapperProvider) { + final var collection = new MongoCollections(mapperProvider, mongoConnection) + .collection(COLLECTION_NAME, ClusterEvent.class) + .withWriteConcern(WriteConcern.JOURNALED); - coll.createIndex(DBSort - .asc("timestamp") - .asc("producer") - .asc("consumers")); + collection.createIndex(Indexes.ascending( + "timestamp", + "producer", + "consumers")); - coll.setWriteConcern(WriteConcern.JOURNALED); - - return coll; + return collection; } @Override @@ -159,24 +130,26 @@ protected Logger getLogger() { @Override public void doRun() { LOG.debug("Opening MongoDB cursor on \"{}\"", COLLECTION_NAME); - try (DBCursor cursor = eventCursor(nodeId)) { + try { + final FindIterable eventsIterable = eventsIterable(nodeId); if (LOG.isTraceEnabled()) { - LOG.trace("MongoDB query plan: {}", cursor.explain()); + LOG.trace("MongoDB query plan: {}", eventsIterable.explain()); } - while (cursor.hasNext()) { - ClusterEvent clusterEvent = cursor.next(); - LOG.trace("Processing cluster event: {}", clusterEvent); + try (final var stream = MongoUtils.stream(eventsIterable)) { + stream.forEach(clusterEvent -> { + LOG.trace("Processing cluster event: {}", clusterEvent); - Object payload = extractPayload(clusterEvent.payload(), clusterEvent.eventClass()); - if (payload != null) { - serverEventBus.post(payload); - } else { - LOG.warn("Couldn't extract payload of cluster event with ID <{}>", clusterEvent.id()); - LOG.debug("Invalid payload in cluster event: {}", clusterEvent); - } + Object payload = extractPayload(clusterEvent.payload(), clusterEvent.eventClass()); + if (payload != null) { + serverEventBus.post(payload); + } else { + LOG.warn("Couldn't extract payload of cluster event with ID <{}>", clusterEvent.id()); + LOG.debug("Invalid payload in cluster event: {}", clusterEvent); + } - updateConsumers(clusterEvent.id(), nodeId); + updateConsumers(clusterEvent.id(), nodeId); + }); } } catch (Exception e) { LOG.warn("Error while reading cluster events from MongoDB, retrying.", e); @@ -194,7 +167,7 @@ public void publishClusterEvent(Object event) { final ClusterEvent clusterEvent = ClusterEvent.create(nodeId.getNodeId(), className, Collections.singleton(nodeId.getNodeId()), event); try { - final String id = dbCollection.save(clusterEvent, WriteConcern.JOURNALED).getSavedId(); + final String id = MongoUtils.insertedIdAsString(collection.insertOne(clusterEvent)); // We are handling a locally generated event, so we can speed up processing by posting it to the local event // bus immediately. Due to having added the local node id to its list of consumers, it will not be picked up // by the db cursor again, avoiding double processing of the event. See #11263 for details. @@ -205,17 +178,13 @@ public void publishClusterEvent(Object event) { } } - private DBCursor eventCursor(NodeId nodeId) { - // Resorting to ugly MongoDB Java Client because of https://github.com/devbliss/mongojack/issues/88 - final BasicDBList consumersList = new BasicDBList(); - consumersList.add(nodeId.getNodeId()); - final DBObject query = new BasicDBObject("consumers", new BasicDBObject("$nin", consumersList)); - - return dbCollection.find(query).sort(DBSort.asc("timestamp")); + private FindIterable eventsIterable(NodeId nodeId) { + return collection.find(Filters.nin("consumers", nodeId.getNodeId())) + .sort(Sorts.ascending("timestamp")); } private void updateConsumers(final String eventId, final NodeId nodeId) { - dbCollection.updateById(eventId, DBUpdate.addToSet("consumers", nodeId.getNodeId())); + collection.updateOne(MongoUtils.idEq(eventId), Updates.addToSet("consumers", nodeId.getNodeId())); } private Object extractPayload(Object payload, String eventClass) { diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java index 7fca212f5f82..856eae555363 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java @@ -38,6 +38,7 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.inputs.transports.NettyTransport; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTime; @@ -49,6 +50,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.Map; +import java.util.Optional; @Codec(name = "gelf", displayName = "GELF") public class GelfCodec extends AbstractCodec { @@ -121,9 +123,8 @@ private static double timestampValue(final JsonNode json) { } } - @Nullable @Override - public Message decode(@Nonnull final RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull final RawMessage rawMessage) { final GELFMessage gelfMessage = new GELFMessage(rawMessage.getPayload(), rawMessage.getRemoteAddress()); final String json = gelfMessage.getJSON(decompressSizeLimit, charset); @@ -135,16 +136,14 @@ public Message decode(@Nonnull final RawMessage rawMessage) { throw new IOException("null result"); } } catch (final Exception e) { - log.error("Could not parse JSON, first 400 characters: " + - StringUtils.abbreviate(json, 403), e); - throw new IllegalStateException("JSON is null/could not be parsed (invalid JSON)", e); + throw InputProcessingException.create("JSON is null/could not be parsed (invalid JSON)", + e, rawMessage, json); } try { validateGELFMessage(node, rawMessage.getId(), rawMessage.getRemoteAddress()); } catch (IllegalArgumentException e) { - log.trace("Invalid GELF message <{}>", node); - throw e; + throw InputProcessingException.create(e.getMessage(), e, rawMessage, json); } // Timestamp. @@ -238,7 +237,7 @@ public Message decode(@Nonnull final RawMessage rawMessage) { message.addField(key, fieldValue); } - return message; + return Optional.of(message); } private void validateGELFMessage(JsonNode jsonNode, UUID id, ResolvableInetSocketAddress remoteAddress) { diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/JsonPathCodec.java b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/JsonPathCodec.java index 9de1d43babdd..f74e46b261be 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/JsonPathCodec.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/JsonPathCodec.java @@ -43,6 +43,7 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; @Codec(name = "jsonpath", displayName = "JSON Path") public class JsonPathCodec extends AbstractCodec { @@ -77,22 +79,25 @@ public JsonPathCodec(@Assisted Configuration configuration, ObjectMapper objectM this.objectMapper = objectMapper; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { Map fields = new HashMap<>(); if (flatten) { final String json = new String(rawMessage.getPayload(), charset); try { fields = flatten(json); } catch (JsonFlattenException e) { - LOG.warn("JSON contains type not supported by flatten method.", e); + throw InputProcessingException.create( + "JSON contains type not supported by flatten method.", e, rawMessage, json); } catch (JsonProcessingException e) { - LOG.warn("Could not parse JSON.", e); + throw InputProcessingException.create( + "Could not parse JSON.", e, rawMessage, json); } } else { if (jsonPath == null) { - return null; + throw InputProcessingException.create( + "Field <%s> is empty for input with id <%s>".formatted(CK_PATH, rawMessage.getSourceNodes().get(0).inputId), + rawMessage); } final String json = new String(rawMessage.getPayload(), charset); fields = read(json); @@ -102,7 +107,7 @@ public Message decode(@Nonnull RawMessage rawMessage) { configuration.getString(CK_SOURCE), rawMessage.getTimestamp()); message.addFields(fields); - return message; + return Optional.of(message); } @VisibleForTesting @@ -137,7 +142,7 @@ protected String buildShortMessage(Map fields) { if (fields.toString().length() > 50) { shortMessage.append(fields.toString().substring(0, 50)).append("[...]"); } else { - shortMessage.append(fields.toString()); + shortMessage.append(fields); } return shortMessage.toString(); diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RandomHttpMessageCodec.java b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RandomHttpMessageCodec.java index b11f6c0e0c1f..6bb152ea93de 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RandomHttpMessageCodec.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RandomHttpMessageCodec.java @@ -29,13 +29,14 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; +import java.util.Optional; import static org.graylog2.inputs.random.generators.FakeHttpRawMessageGenerator.GeneratorState; @@ -53,22 +54,21 @@ public RandomHttpMessageCodec(@Assisted Configuration configuration, ObjectMappe this.messageFactory = messageFactory; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { if (!rawMessage.getCodecName().equals(getName())) { - log.error("Cannot decode payload type {}, skipping message {}", - rawMessage.getCodecName(), rawMessage.getId()); - return null; + throw InputProcessingException.create( + "Cannot decode payload type %s, skipping message %s".formatted(rawMessage.getCodecName(), rawMessage.getId()), + rawMessage, new String(rawMessage.getPayload(), charset)); } try { final GeneratorState state = objectMapper.readValue(rawMessage.getPayload(), GeneratorState.class); - final Message message = FakeHttpRawMessageGenerator.generateMessage(messageFactory, state); - return message; - } catch (IOException e) { - log.error("Cannot decode message to class FakeHttpRawMessageGenerator.GeneratorState", e); + return Optional.of(FakeHttpRawMessageGenerator.generateMessage(messageFactory, state)); + } catch (Exception e) { + throw InputProcessingException.create( + "Cannot decode message to class FakeHttpRawMessageGenerator.GeneratorState", + rawMessage, new String(rawMessage.getPayload(), charset)); } - return null; } @Nullable diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RawCodec.java b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RawCodec.java index ef60bafcaf15..e9851e3d4f46 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RawCodec.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/RawCodec.java @@ -33,6 +33,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Optional; @Codec(name = "raw", displayName = "Raw String") public class RawCodec extends AbstractCodec { @@ -45,10 +46,9 @@ public RawCodec(@Assisted Configuration configuration, MessageFactory messageFac this.messageFactory = messageFactory; } - @Nullable @Override - public Message decode(@Nonnull RawMessage raw) { - return messageFactory.createMessage(new String(raw.getPayload(), charset), null, raw.getTimestamp()); + public Optional decodeSafe(@Nonnull RawMessage raw) { + return Optional.of(messageFactory.createMessage(new String(raw.getPayload(), charset), null, raw.getTimestamp())); } @Nullable diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/SyslogCodec.java b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/SyslogCodec.java index f1032f7e9e36..638ae0dcb9ec 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/SyslogCodec.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/SyslogCodec.java @@ -38,6 +38,7 @@ import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; import org.graylog2.plugin.inputs.codecs.CodecAggregator; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.inputs.transports.NettyTransport; import org.graylog2.plugin.journal.RawMessage; import org.graylog2.syslog4j.server.SyslogServerEventIF; @@ -59,6 +60,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.regex.Pattern; import static com.codahale.metrics.MetricRegistry.name; @@ -89,9 +91,8 @@ public SyslogCodec(@Assisted Configuration configuration, MetricRegistry metricR this.messageFactory = messageFactory; } - @Nullable @Override - public Message decode(@Nonnull RawMessage rawMessage) { + public Optional decodeSafe(@Nonnull RawMessage rawMessage) { final String msg = new String(rawMessage.getPayload(), charset); try (Timer.Context ignored = this.decodeTime.time()) { final ResolvableInetSocketAddress address = rawMessage.getRemoteAddress(); @@ -101,10 +102,13 @@ public Message decode(@Nonnull RawMessage rawMessage) { } else { remoteAddress = address.getInetSocketAddress(); } - return parse(msg, remoteAddress == null ? null : remoteAddress.getAddress(), rawMessage.getTimestamp()); + return Optional.of(parse(msg, remoteAddress == null ? null : remoteAddress.getAddress(), rawMessage.getTimestamp())); + } catch (Exception e) { + throw InputProcessingException.create("Could not deserialize Syslog message.", e, rawMessage, msg); } } + @Nonnull private Message parse(String msg, InetAddress remoteAddress, DateTime receivedTimestamp) { /* * ZOMG funny 80s neckbeard protocols. We are now deciding if to parse diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java index de87cb469f0e..b99f13a94cae 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/Codec.java @@ -24,10 +24,22 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Optional; public interface Codec { + + default Optional decodeSafe(@Nonnull RawMessage rawMessage) { + return Optional.ofNullable(decode(rawMessage)); + } + + /** + * @deprecated use {@link #decodeSafe(RawMessage)} instead. + */ + @Deprecated @Nullable - Message decode(@Nonnull RawMessage rawMessage); + default Message decode(@Nonnull RawMessage rawMessage) { + throw new UnsupportedOperationException("implement decodeSafe method"); + } @Nullable CodecAggregator getAggregator(); @@ -39,7 +51,9 @@ public interface Codec { interface Factory { C create(Configuration configuration); + Config getConfig(); + Descriptor getDescriptor(); } @@ -48,6 +62,7 @@ interface Config { String CK_CHARSET_NAME = "charset_name"; ConfigurationRequest getRequestedConfiguration(); + void overrideDefaultValues(@Nonnull ConfigurationRequest cr); } diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/NullCodec.java b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/NullCodec.java index 774c601ecc6a..ae3803008a60 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/NullCodec.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/codecs/NullCodec.java @@ -25,6 +25,7 @@ import org.graylog2.plugin.journal.RawMessage; import javax.annotation.Nullable; +import java.util.Optional; /** * This codec always returns a null Message. @@ -33,10 +34,9 @@ public class NullCodec implements Codec { public static final String NAME = "NullCodec"; - @Nullable @Override - public Message decode(@NonNull RawMessage rawMessage) { - return null; + public Optional decodeSafe(@NonNull RawMessage rawMessage) { + return Optional.empty(); } @Nullable diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/failure/InputProcessingException.java b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/failure/InputProcessingException.java new file mode 100644 index 000000000000..f475043b57be --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/failure/InputProcessingException.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.plugin.inputs.failure; + + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import org.graylog2.plugin.journal.RawMessage; + +import java.util.Optional; + +public class InputProcessingException extends RuntimeException { + + + private final InputProcessingFailure inputProcessingFailure; + + public InputProcessingException(InputProcessingFailure inputProcessingFailure) { + super(inputProcessingFailure.errorMessage(), inputProcessingFailure.exception()); + this.inputProcessingFailure = inputProcessingFailure; + } + + public Optional inputMessageString() { + return Optional.ofNullable(inputProcessingFailure.inputMessage()); + } + + public static InputProcessingException create(@Nonnull String errorMessage, + @Nullable Throwable throwable, + @Nonnull RawMessage rawMessage, + @Nullable String inputMessage) { + return new InputProcessingException(new InputProcessingFailure(errorMessage, throwable, rawMessage, inputMessage)); + } + + public static InputProcessingException create(@Nonnull String errorMessage, + @Nonnull RawMessage rawMessage, + @Nullable String inputMessage) { + return new InputProcessingException(new InputProcessingFailure(errorMessage, null, rawMessage, inputMessage)); + } + + public static InputProcessingException create(@Nonnull String errorMessage, + @Nullable Exception exception, + @Nonnull RawMessage rawMessage) { + return new InputProcessingException(new InputProcessingFailure(errorMessage, exception, rawMessage, null)); + } + + public static InputProcessingException create(@Nonnull String errorMessage, + @Nonnull RawMessage rawMessage) { + return new InputProcessingException(new InputProcessingFailure(errorMessage, null, rawMessage, null)); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inputs/failure/InputProcessingFailure.java b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/failure/InputProcessingFailure.java new file mode 100644 index 000000000000..8dccec293fdb --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inputs/failure/InputProcessingFailure.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.plugin.inputs.failure; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import org.graylog2.plugin.journal.RawMessage; + +public record InputProcessingFailure(@Nonnull String errorMessage, + @Nullable Throwable exception, + @Nonnull RawMessage rawMessage, + @Nullable String inputMessage) { +} diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/messages/MessageResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/messages/MessageResource.java index 6f7825b6b639..e2eec86eeaa2 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/messages/MessageResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/messages/MessageResource.java @@ -63,6 +63,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.Optional; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; @@ -161,17 +162,14 @@ public ResultMessage parse(@ApiParam(name = "JSON body", required = true) Messag } private Message decodeMessage(Codec codec, ResolvableInetSocketAddress remoteAddress, RawMessage rawMessage) { - Message message; + Optional messageOpt; try { - message = codec.decode(rawMessage); - + messageOpt = codec.decodeSafe(rawMessage); } catch (Exception e) { throw new BadRequestException("Could not decode message"); } - if (message == null) { - throw new BadRequestException("Could not decode message"); - } + Message message = messageOpt.orElseThrow(() -> new BadRequestException("Could not decode message")); // Ensure the decoded Message has a source, otherwise creating a ResultMessage will fail if (isNullOrEmpty(message.getSource())) { diff --git a/graylog2-server/src/main/java/org/graylog2/shared/buffers/processors/DecodingProcessor.java b/graylog2-server/src/main/java/org/graylog2/shared/buffers/processors/DecodingProcessor.java index d9990786d2e1..22a6a696bcd7 100644 --- a/graylog2-server/src/main/java/org/graylog2/shared/buffers/processors/DecodingProcessor.java +++ b/graylog2-server/src/main/java/org/graylog2/shared/buffers/processors/DecodingProcessor.java @@ -33,6 +33,7 @@ import org.graylog2.plugin.buffers.MessageEvent; import org.graylog2.plugin.inputs.codecs.Codec; import org.graylog2.plugin.inputs.codecs.MultiMessageCodec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.graylog2.shared.journal.Journal; import org.graylog2.shared.messageq.MessageQueueAcknowledger; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -118,7 +120,7 @@ public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throw } } - private void processMessage(final MessageEvent event) throws ExecutionException { + private void processMessage(final MessageEvent event) { final RawMessage raw = event.getRaw(); // for backwards compatibility: the last source node should contain the input we use. @@ -142,7 +144,7 @@ private void processMessage(final MessageEvent event) throws ExecutionException final Codec codec = factory.create(raw.getCodecConfig()); final String baseMetricName = name(codec.getClass(), inputIdOnCurrentNode); - Message message = null; + Optional message = Optional.empty(); Collection messages = null; final Timer.Context decodeTimeCtx = parseTime.time(); @@ -153,8 +155,16 @@ private void processMessage(final MessageEvent event) throws ExecutionException if (codec instanceof MultiMessageCodec) { messages = ((MultiMessageCodec) codec).decodeMessages(raw); } else { - message = codec.decode(raw); + message = codec.decodeSafe(raw); } + } catch (InputProcessingException e) { + if(LOG.isTraceEnabled() && e.inputMessageString().isPresent()) { + LOG.error("%s - input message: %s".formatted(e.getMessage(), e.inputMessageString().get()), e.getCause()); + }else{ + LOG.error(e.getMessage(), e.getCause()); + } + metricRegistry.meter(name(baseMetricName, "failures")).mark(); + throw e; } catch (RuntimeException e) { LOG.error("Unable to decode raw message {} on input <{}>.", raw, inputIdOnCurrentNode); metricRegistry.meter(name(baseMetricName, "failures")).mark(); @@ -163,8 +173,8 @@ private void processMessage(final MessageEvent event) throws ExecutionException decodeTime = decodeTimeCtx.stop(); } - if (message != null) { - event.setMessage(postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, message, decodeTime)); + if (message.isPresent()) { + event.setMessage(postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, message.get(), decodeTime)); } else if (messages != null && !messages.isEmpty()) { final List processedMessages = Lists.newArrayListWithCapacity(messages.size()); diff --git a/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java b/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java index e95994849e5b..a556b35c7d23 100644 --- a/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java @@ -67,7 +67,7 @@ public void testAdditionalEventDataField() { "\"eventType\": \"AwsConsoleSignIn\",\n" + "\"recipientAccountId\": \"1111122221111\"\n" + "}").getBytes(StandardCharsets.UTF_8)); - Message message = codec.decode(rawMessage); + Message message = codec.decodeSafe(rawMessage).get(); String additional_event_data = message.getField("additional_event_data").toString(); assertTrue(additional_event_data.contains("MFAUsed=Yes")); @@ -104,7 +104,7 @@ public void testNoAdditionalEventDataField() { "\"eventType\": \"AwsConsoleSignIn\",\n" + "\"recipientAccountId\": \"1111122221111\"\n" + "}").getBytes(StandardCharsets.UTF_8)); - Message message = codec.decode(rawMessage); + Message message = codec.decodeSafe(rawMessage).get(); assertNull(message.getField("additional_event_data")); } } diff --git a/graylog2-server/src/test/java/org/graylog/events/processor/DBEventProcessorStateServiceTest.java b/graylog2-server/src/test/java/org/graylog/events/processor/DBEventProcessorStateServiceTest.java index 51ab1356c902..ea0b8d4e4a25 100644 --- a/graylog2-server/src/test/java/org/graylog/events/processor/DBEventProcessorStateServiceTest.java +++ b/graylog2-server/src/test/java/org/graylog/events/processor/DBEventProcessorStateServiceTest.java @@ -20,6 +20,7 @@ import org.graylog.testing.mongodb.MongoDBFixtures; import org.graylog.testing.mongodb.MongoDBInstance; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoCollections; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -46,7 +47,7 @@ public class DBEventProcessorStateServiceTest { @Before public void setUp() { - stateService = new DBEventProcessorStateService(mongodb.mongoConnection(), objectMapperProvider); + stateService = new DBEventProcessorStateService(new MongoCollections(objectMapperProvider, mongodb.mongoConnection())); } @Test diff --git a/graylog2-server/src/test/java/org/graylog/events/processor/EventProcessorDependencyCheckTest.java b/graylog2-server/src/test/java/org/graylog/events/processor/EventProcessorDependencyCheckTest.java index 165c4b30ed0c..739264a22ff0 100644 --- a/graylog2-server/src/test/java/org/graylog/events/processor/EventProcessorDependencyCheckTest.java +++ b/graylog2-server/src/test/java/org/graylog/events/processor/EventProcessorDependencyCheckTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import org.graylog.testing.mongodb.MongoDBInstance; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoCollections; import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; @@ -52,7 +53,7 @@ public class EventProcessorDependencyCheckTest { @Before public void setUp() throws Exception { - stateService = new DBEventProcessorStateService(mongodb.mongoConnection(), objectMapperProvider); + stateService = new DBEventProcessorStateService(new MongoCollections(objectMapperProvider, mongodb.mongoConnection())); dependencyCheck = new EventProcessorDependencyCheck(stateService, dbProcessingStatusService); } diff --git a/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/AWSCodecTest.java b/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/AWSCodecTest.java index 4deb56c80328..ba22558008d3 100644 --- a/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/AWSCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/AWSCodecTest.java @@ -54,7 +54,7 @@ public void testKinesisFlowLogCodec() throws JsonProcessingException { KinesisLogEntry kinesisLogEntry = KinesisLogEntry.create("a-stream", "log-group", "log-stream", timestamp, "2 423432432432 eni-3244234 172.1.1.2 172.1.1.2 80 2264 6 1 52 1559738144 1559738204 ACCEPT OK"); - Message message = codec.decode(new RawMessage(objectMapper.writeValueAsBytes(kinesisLogEntry))); + Message message = codec.decodeSafe(new RawMessage(objectMapper.writeValueAsBytes(kinesisLogEntry))).get(); Assert.assertEquals("log-group", message.getField(AbstractKinesisCodec.FIELD_LOG_GROUP)); Assert.assertEquals("log-stream", message.getField(AbstractKinesisCodec.FIELD_LOG_STREAM)); Assert.assertEquals("a-stream", message.getField(AbstractKinesisCodec.FIELD_KINESIS_STREAM)); @@ -89,7 +89,7 @@ public void testKinesisRawCodec() throws JsonProcessingException { final KinesisLogEntry kinesisLogEntry = KinesisLogEntry.create("a-stream", "log-group", "log-stream", timestamp, "This a raw message"); - Message message = codec.decode(new RawMessage(objectMapper.writeValueAsBytes(kinesisLogEntry))); + Message message = codec.decodeSafe(new RawMessage(objectMapper.writeValueAsBytes(kinesisLogEntry))).get(); Assert.assertEquals("log-group", message.getField(AbstractKinesisCodec.FIELD_LOG_GROUP)); Assert.assertEquals("log-stream", message.getField(AbstractKinesisCodec.FIELD_LOG_STREAM)); Assert.assertEquals("a-stream", message.getField(AbstractKinesisCodec.FIELD_KINESIS_STREAM)); @@ -97,4 +97,4 @@ public void testKinesisRawCodec() throws JsonProcessingException { Assert.assertEquals("This a raw message", message.getField("message")); Assert.assertEquals(timestamp, message.getTimestamp()); } -} \ No newline at end of file +} diff --git a/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/CloudWatchFlowLogCodecTest.java b/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/CloudWatchFlowLogCodecTest.java index e13f28a6a9f1..4cd8f5ef3be7 100644 --- a/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/CloudWatchFlowLogCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/integrations/aws/codecs/CloudWatchFlowLogCodecTest.java @@ -50,7 +50,7 @@ public void testFlowLogCodecValues() { final DateTime timestamp = DateTime.now(DateTimeZone.UTC); final KinesisLogEntry logEvent = KinesisLogEntry.create("a-stream", "log-group", "log-stream", timestamp, flowLogMessage); - final Message message = codec.decodeLogData(logEvent); + final Message message = codec.decodeLogData(logEvent).get(); Assert.assertEquals("log-group", message.getField(AbstractKinesisCodec.FIELD_LOG_GROUP)); Assert.assertEquals("log-stream", message.getField(AbstractKinesisCodec.FIELD_LOG_STREAM)); Assert.assertEquals("a-stream", message.getField(AbstractKinesisCodec.FIELD_KINESIS_STREAM)); diff --git a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodecTest.java b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodecTest.java index bbe4686a1ba0..53ce195fafd5 100644 --- a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto/PaloAltoCodecTest.java @@ -75,16 +75,16 @@ public void testAllSyslogFormats() { PaloAltoCodec codec = new PaloAltoCodec(Configuration.EMPTY_CONFIGURATION, messageFactory); - Message message = codec.decode(new RawMessage(SYSLOG_THREAT_MESSAGE.getBytes(StandardCharsets.UTF_8))); + Message message = codec.decodeSafe(new RawMessage(SYSLOG_THREAT_MESSAGE.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("THREAT", message.getField("type")); - message = codec.decode(new RawMessage(SYSLOG_THREAT_MESSAGE_DOUBLE_SPACE_DATE.getBytes(StandardCharsets.UTF_8))); + message = codec.decodeSafe(new RawMessage(SYSLOG_THREAT_MESSAGE_DOUBLE_SPACE_DATE.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("THREAT", message.getField("type")); - message = codec.decode(new RawMessage(SYSLOG_THREAT_MESSAGE_NO_HOST.getBytes(StandardCharsets.UTF_8))); + message = codec.decodeSafe(new RawMessage(SYSLOG_THREAT_MESSAGE_NO_HOST.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("THREAT", message.getField("type")); - message = codec.decode(new RawMessage(SYSLOG_THREAT_MESSAGE_NO_HOST_DOUBLE_SPACE_DATE.getBytes(StandardCharsets.UTF_8))); + message = codec.decodeSafe(new RawMessage(SYSLOG_THREAT_MESSAGE_NO_HOST_DOUBLE_SPACE_DATE.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("THREAT", message.getField("type")); } @@ -93,11 +93,11 @@ public void testMessageWithLineBreak() { // Verify that a messages with a line break at the end does not break parsing. PaloAltoCodec codec = new PaloAltoCodec(Configuration.EMPTY_CONFIGURATION, messageFactory); - Message message = codec.decode(new RawMessage(PANORAMA_WITH_LINE_BREAK.getBytes(StandardCharsets.UTF_8))); + Message message = codec.decodeSafe(new RawMessage(PANORAMA_WITH_LINE_BREAK.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("SYSTEM", message.getField("type")); codec = new PaloAltoCodec(Configuration.EMPTY_CONFIGURATION, messageFactory); - message = codec.decode(new RawMessage(SYSLOG_WITH_LINE_BREAK.getBytes(StandardCharsets.UTF_8))); + message = codec.decodeSafe(new RawMessage(SYSLOG_WITH_LINE_BREAK.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("THREAT", message.getField("type")); } @@ -107,7 +107,7 @@ public void testMoreSyslogFormats() { // Test an extra list of messages. for (String threatString : MORE_SYSLOG_THREAT_MESSAGES) { PaloAltoCodec codec = new PaloAltoCodec(Configuration.EMPTY_CONFIGURATION, messageFactory); - Message message = codec.decode(new RawMessage(threatString.getBytes(StandardCharsets.UTF_8))); + Message message = codec.decodeSafe(new RawMessage(threatString.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("THREAT", message.getField("type")); } } @@ -117,7 +117,7 @@ public void syslogValuesTest() { // Test System message results PaloAltoCodec codec = new PaloAltoCodec(Configuration.EMPTY_CONFIGURATION, messageFactory); - Message message = codec.decode(new RawMessage(SYSLOG_THREAT_MESSAGE_NO_HOST_DOUBLE_SPACE_DATE.getBytes(StandardCharsets.UTF_8))); + Message message = codec.decodeSafe(new RawMessage(SYSLOG_THREAT_MESSAGE_NO_HOST_DOUBLE_SPACE_DATE.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("THREAT", message.getField("type")); } @@ -126,7 +126,7 @@ public void valuesTest() { // Test System message results PaloAltoCodec codec = new PaloAltoCodec(Configuration.EMPTY_CONFIGURATION, messageFactory); - Message message = codec.decode(new RawMessage(PANORAMA_SYSTEM_MESSAGE.getBytes(StandardCharsets.UTF_8))); + Message message = codec.decodeSafe(new RawMessage(PANORAMA_SYSTEM_MESSAGE.getBytes(StandardCharsets.UTF_8))).get(); assertEquals("SYSTEM", message.getField("type")); assertEquals(message.getField("module"), "general"); @@ -144,7 +144,7 @@ public void valuesTest() { assertEquals(0, ((DateTime) message.getField("timestamp")).compareTo(new DateTime("2018-09-19T11:50:35.000-05:00", DateTimeZone.UTC))); // Test Traffic message results - message = codec.decode(new RawMessage(PANORAMA_TRAFFIC_MESSAGE.getBytes(StandardCharsets.UTF_8))); + message = codec.decodeSafe(new RawMessage(PANORAMA_TRAFFIC_MESSAGE.getBytes(StandardCharsets.UTF_8))).get(); assertEquals(message.getField("bytes_received"), 140L); assertEquals(message.getField("source"), "Panorama--2"); assertEquals(message.getField("repeat_count"), 1L); diff --git a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodecTest.java b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodecTest.java index 2151f9524f6d..3c3b723d45fe 100644 --- a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto11/PaloAlto11xCodecTest.java @@ -109,7 +109,7 @@ private void givenStoreFullMessage(boolean storeFullMessage) { // WHENs private void whenDecodeIsCalled() { - out = cut.decode(in); + out = cut.decodeSafe(in).get(); } private void thenOutputMessageContainsExpectedFields(boolean shouldContainFullMessage) { diff --git a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodecTest.java b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodecTest.java index 4cab7269266c..8a7deee81a04 100644 --- a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xCodecTest.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.assertj.core.api.Assertions; import org.graylog.integrations.inputs.paloalto.PaloAltoMessageBase; import org.graylog.integrations.inputs.paloalto.PaloAltoMessageType; import org.graylog.integrations.inputs.paloalto.PaloAltoParser; @@ -25,6 +26,7 @@ import org.graylog2.plugin.MessageFactory; import org.graylog2.plugin.TestMessageFactory; import org.graylog2.plugin.configuration.Configuration; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -213,9 +215,7 @@ public void decode_returnsNull_whenRawPaloParseFails() { givenGoodInputRawMessage(); givenRawParserReturnsNull(); - whenDecodeIsCalled(); - - thenOutputMessageIsNull(); + Assertions.assertThatThrownBy(this::whenDecodeIsCalled).isInstanceOf(InputProcessingException.class); } // GIVENs @@ -243,7 +243,7 @@ private void givenGoodFieldProducer() { // WHENs private void whenDecodeIsCalled() { - out = cut.decode(in); + out = cut.decodeSafe(in).get(); } // THENs @@ -271,8 +271,4 @@ private void thenOutputMessageContainsExpectedFields(boolean shouldContainFullMe assertThat(out.getField(Message.FIELD_FULL_MESSAGE), nullValue()); } } - - private void thenOutputMessageIsNull() { - assertThat(out, nullValue()); - } } diff --git a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xTemplatesTest.java b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xTemplatesTest.java index e9142fe91972..ff48961f3e16 100644 --- a/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xTemplatesTest.java +++ b/graylog2-server/src/test/java/org/graylog/integrations/inputs/paloalto9/PaloAlto9xTemplatesTest.java @@ -78,7 +78,7 @@ public void verifyConfigurationMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -114,7 +114,7 @@ public void verifyCorrelationMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -149,7 +149,7 @@ public void verifyGlobalProtectMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -196,7 +196,7 @@ public void verifyGlobalProtect913MessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -249,7 +249,7 @@ public void verifyHipMatchMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -291,7 +291,7 @@ public void verifySystemMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -326,7 +326,7 @@ public void verifyThreatMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -478,7 +478,7 @@ public void verifyThreatMessageParsing_withRepeatedSameXFF() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -494,7 +494,7 @@ public void verifyThreatMessageParsing_withDifferentXFF() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -513,7 +513,7 @@ public void verifyTrafficMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); @@ -653,7 +653,7 @@ public void verifyUserIdMessageParsing() { String rawMessage = SYSLOG_PREFIX + log; RawMessage in = new RawMessage(rawMessage.getBytes(StandardCharsets.UTF_8)); - Message out = cut.decode(in); + Message out = cut.decodeSafe(in).get(); assertThat(out, notNullValue()); assertThat(out.getField(Message.FIELD_FULL_MESSAGE), is(rawMessage)); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java b/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java index 6694a1f88cd4..4f5b2877fe64 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java @@ -22,6 +22,7 @@ import org.graylog2.plugin.MessageFactory; import org.graylog2.plugin.TestMessageFactory; import org.graylog2.plugin.configuration.Configuration; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; import org.joda.time.DateTime; @@ -38,6 +39,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class Beats2CodecTest { @Rule @@ -55,8 +57,9 @@ public void setUp() throws Exception { } @Test - public void decodeReturnsNullIfPayloadCouldNotBeDecoded() throws Exception { - assertThat(codec.decode(new RawMessage(new byte[0]))).isNull(); + public void decodeReturnsNullIfPayloadCouldNotBeDecoded() { + assertThatThrownBy(() -> codec.decodeSafe(new RawMessage(new byte[0]))) + .isInstanceOf(InputProcessingException.class); } @Test @@ -64,7 +67,7 @@ public void decodeMessagesHandlesFilebeatMessagesWithoutPrefix() throws Exceptio configuration = new Configuration(Collections.singletonMap("no_beats_prefix", true)); codec = new Beats2Codec(configuration, objectMapper, messageFactory); - final Message message = codec.decode(messageFromJson("filebeat.json")); + final Message message = codec.decodeSafe(messageFromJson("filebeat.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("TEST"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -81,7 +84,7 @@ public void decodeMessagesHandlesFilebeatMessagesWithoutPrefix() throws Exceptio @Test public void decodeMessagesHandlesFilebeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("filebeat.json")); + final Message message = codec.decodeSafe(messageFromJson("filebeat.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("TEST"); assertThat(message.getSource()).isEqualTo("example.local"); @@ -100,7 +103,7 @@ public void decodeMessagesHandlesFilebeatMessages() throws Exception { @Test public void decodeMessagesHandlesPacketbeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("packetbeat-dns.json")); + final Message message = codec.decodeSafe(messageFromJson("packetbeat-dns.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -114,7 +117,7 @@ public void decodeMessagesHandlesPacketbeatMessages() throws Exception { @Test public void decodeMessagesHandlesPacketbeatV8Messages() throws Exception { - final Message message = codec.decode(messageFromJson("packetbeat-mongodb-v8.json")); + final Message message = codec.decodeSafe(messageFromJson("packetbeat-mongodb-v8.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2022, 11, 7, 9, 26, 10, 579, DateTimeZone.UTC)); @@ -132,7 +135,7 @@ public void decodeMessagesHandlesPacketbeatV8Messages() throws Exception { @Test public void decodeMessagesHandlesTopbeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("topbeat-system.json")); + final Message message = codec.decodeSafe(messageFromJson("topbeat-system.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -142,7 +145,7 @@ public void decodeMessagesHandlesTopbeatMessages() throws Exception { @Test public void decodeMessagesHandlesWinlogbeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("winlogbeat.json")); + final Message message = codec.decodeSafe(messageFromJson("winlogbeat.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 11, 24, 12, 13, DateTimeZone.UTC)); @@ -156,7 +159,7 @@ public void decodeMessagesHandlesWinlogbeatMessages() throws Exception { @Test public void decodeMessagesHandlesWinlogbeatv7Messages() throws Exception { - final Message message = codec.decode(messageFromJson("winlogbeat-v7.json")); + final Message message = codec.decodeSafe(messageFromJson("winlogbeat-v7.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 11, 24, 12, 13, DateTimeZone.UTC)); @@ -169,7 +172,7 @@ public void decodeMessagesHandlesWinlogbeatv7Messages() throws Exception { @Test public void decodeMessagesHandleGenericBeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("generic.json")); + final Message message = codec.decodeSafe(messageFromJson("generic.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("unknown"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -179,7 +182,7 @@ public void decodeMessagesHandleGenericBeatMessages() throws Exception { @Test public void decodeMessagesHandleGenericBeatMessagesWithFields() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-fields.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-fields.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("unknown"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -210,7 +213,7 @@ public void decodeMessagesHandlesMetricbeatMessages() throws Exception { }; for (String testFile : testFiles) { - final Message message = codec.decode(messageFromJson(testFile)); + final Message message = codec.decodeSafe(messageFromJson(testFile)).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 12, 14, 12, 0, DateTimeZone.UTC)); @@ -220,7 +223,7 @@ public void decodeMessagesHandlesMetricbeatMessages() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithDocker() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-docker.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-docker.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("-"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -234,7 +237,7 @@ public void decodeMessagesHandlesGenericBeatWithDocker() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithKubernetes() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-kubernetes.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-kubernetes.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("-"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -248,7 +251,7 @@ public void decodeMessagesHandlesGenericBeatWithKubernetes() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithCloudAlibaba() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-alibaba.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-alibaba.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("-"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -263,7 +266,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudAlibaba() throws Exception @Test public void decodeMessagesHandlesGenericBeatWithCloudDigitalOcean() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-digital-ocean.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-digital-ocean.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("-"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -277,7 +280,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudDigitalOcean() throws Excep @Test public void decodeMessagesHandlesGenericBeatWithCloudEC2() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-ec2.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-ec2.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("-"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -293,7 +296,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudEC2() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithCloudGCE() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-gce.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-gce.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("-"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -309,7 +312,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudGCE() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithCloudTencent() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-tencent.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-tencent.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("-"); assertThat(message.getSource()).isEqualTo("unknown"); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/beats/BeatsCodecTest.java b/graylog2-server/src/test/java/org/graylog/plugins/beats/BeatsCodecTest.java index 4429a8ff2769..f68f4aa83017 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/beats/BeatsCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/beats/BeatsCodecTest.java @@ -22,6 +22,7 @@ import org.graylog2.plugin.MessageFactory; import org.graylog2.plugin.TestMessageFactory; import org.graylog2.plugin.configuration.Configuration; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; import org.joda.time.DateTime; @@ -38,6 +39,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class BeatsCodecTest { @Rule @@ -56,13 +58,14 @@ public void setUp() throws Exception { } @Test - public void decodeReturnsNullIfPayloadCouldNotBeDecoded() throws Exception { - assertThat(codec.decode(new RawMessage(new byte[0]))).isNull(); + public void decodeReturnsNullIfPayloadCouldNotBeDecoded() { + assertThatThrownBy(() -> codec.decodeSafe(new RawMessage(new byte[0]))) + .isInstanceOf(InputProcessingException.class); } @Test public void decodeMessagesHandlesFilebeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("filebeat.json")); + final Message message = codec.decodeSafe(messageFromJson("filebeat.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("TEST"); assertThat(message.getSource()).isEqualTo("example.local"); @@ -79,7 +82,7 @@ public void decodeMessagesHandlesFilebeatMessages() throws Exception { @Test public void decodeMessagesHandlesPacketbeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("packetbeat-dns.json")); + final Message message = codec.decodeSafe(messageFromJson("packetbeat-dns.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -89,7 +92,7 @@ public void decodeMessagesHandlesPacketbeatMessages() throws Exception { @Test public void decodeMessagesHandlesTopbeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("topbeat-system.json")); + final Message message = codec.decodeSafe(messageFromJson("topbeat-system.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -99,7 +102,7 @@ public void decodeMessagesHandlesTopbeatMessages() throws Exception { @Test public void decodeMessagesHandlesWinlogbeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("winlogbeat.json")); + final Message message = codec.decodeSafe(messageFromJson("winlogbeat.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 11, 24, 12, 13, DateTimeZone.UTC)); @@ -113,7 +116,7 @@ public void decodeMessagesHandlesWinlogbeatMessages() throws Exception { @Test public void decodeMessagesHandleGenericBeatMessages() throws Exception { - final Message message = codec.decode(messageFromJson("generic.json")); + final Message message = codec.decodeSafe(messageFromJson("generic.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("unknown"); @@ -124,7 +127,7 @@ public void decodeMessagesHandleGenericBeatMessages() throws Exception { @Test public void decodeMessagesHandleGenericBeatMessagesWithFields() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-fields.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-fields.json")).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("unknown"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC)); @@ -155,7 +158,7 @@ public void decodeMessagesHandlesMetricbeatMessages() throws Exception { }; for (String testFile : testFiles) { - final Message message = codec.decode(messageFromJson(testFile)); + final Message message = codec.decodeSafe(messageFromJson(testFile)).get(); assertThat(message).isNotNull(); assertThat(message.getSource()).isEqualTo("example.local"); assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 12, 14, 12, 0, DateTimeZone.UTC)); @@ -165,7 +168,7 @@ public void decodeMessagesHandlesMetricbeatMessages() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithDocker() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-docker.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-docker.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("null"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -179,7 +182,7 @@ public void decodeMessagesHandlesGenericBeatWithDocker() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithKubernetes() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-kubernetes.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-kubernetes.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("null"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -193,7 +196,7 @@ public void decodeMessagesHandlesGenericBeatWithKubernetes() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithCloudAlibaba() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-alibaba.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-alibaba.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("null"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -208,7 +211,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudAlibaba() throws Exception @Test public void decodeMessagesHandlesGenericBeatWithCloudDigitalOcean() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-digital-ocean.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-digital-ocean.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("null"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -222,7 +225,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudDigitalOcean() throws Excep @Test public void decodeMessagesHandlesGenericBeatWithCloudEC2() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-ec2.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-ec2.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("null"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -238,7 +241,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudEC2() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithCloudGCE() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-gce.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-gce.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("null"); assertThat(message.getSource()).isEqualTo("unknown"); @@ -254,7 +257,7 @@ public void decodeMessagesHandlesGenericBeatWithCloudGCE() throws Exception { @Test public void decodeMessagesHandlesGenericBeatWithCloudTencent() throws Exception { - final Message message = codec.decode(messageFromJson("generic-with-cloud-tencent.json")); + final Message message = codec.decodeSafe(messageFromJson("generic-with-cloud-tencent.json")).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("null"); assertThat(message.getSource()).isEqualTo("unknown"); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/beats/ConsolePrinter.java b/graylog2-server/src/test/java/org/graylog/plugins/beats/ConsolePrinter.java index 93ddbb174599..d832f92863fc 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/beats/ConsolePrinter.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/beats/ConsolePrinter.java @@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception { .channel(NioServerSocketChannel.class) .childHandler(new ChannelInitializer() { @Override - public void initChannel(SocketChannel ch) throws Exception { + public void initChannel(SocketChannel ch) { ch.pipeline().addLast("logging", new LoggingHandler()); ch.pipeline().addLast("beats-frame-decoder", new BeatsFrameDecoder()); ch.pipeline().addLast("beats-codec", new BeatsCodecHandler()); @@ -77,13 +77,13 @@ public static class BeatsCodecHandler extends SimpleChannelInboundHandler() { @Override - public void initChannel(SocketChannel ch) throws Exception { + public void initChannel(SocketChannel ch) { ch.pipeline().addLast("logging", new LoggingHandler()); ch.pipeline().addLast("beats-frame-decoder", new BeatsFrameDecoder()); ch.pipeline().addLast("beats-deprecated-codec", new BeatsCodecHandler()); @@ -77,13 +77,13 @@ public static class BeatsCodecHandler extends SimpleChannelInboundHandler codec.decode(new RawMessage(new byte[0]))) + .isThrownBy(() -> codec.decodeSafe(new RawMessage(new byte[0]))) .withMessage("MultiMessageCodec " + NetFlowCodec.class + " does not support decode()"); } @@ -116,8 +118,7 @@ public void decodeMessagesReturnsNullIfMessageWasInvalid() throws Exception { final InetSocketAddress source = new InetSocketAddress(InetAddress.getLocalHost(), 12345); final RawMessage rawMessage = new RawMessage(b, source); - final Collection messages = codec.decodeMessages(rawMessage); - assertThat(messages).isNull(); + assertThatThrownBy(() -> codec.decodeMessages(rawMessage)).isInstanceOf(InputProcessingException.class); } @Test @@ -136,8 +137,7 @@ public byte[] getPayload() { } }; - final Collection messages = codec.decodeMessages(rawMessage); - assertThat(messages).isNull(); + assertThatThrownBy(() -> codec.decodeMessages(rawMessage)).isInstanceOf(InputProcessingException.class); } @Test @@ -145,6 +145,7 @@ public void decodeMessagesThrowsEmptyTemplateExceptionWithIncompleteNetFlowV9() final byte[] b = Resources.toByteArray(Resources.getResource("netflow-data/netflow-v9-3_incomplete.dat")); final InetSocketAddress source = new InetSocketAddress(InetAddress.getLocalHost(), 12345); - assertThat(codec.decodeMessages(new RawMessage(b, source))).isNull(); + assertThatThrownBy(() -> codec.decodeMessages(new RawMessage(b, source))).isInstanceOf(InputProcessingException.class); + } } diff --git a/graylog2-server/src/test/java/org/graylog2/events/ClusterEventCleanupPeriodicalTest.java b/graylog2-server/src/test/java/org/graylog2/events/ClusterEventCleanupPeriodicalTest.java index 3b2d085dc340..2ba887270f55 100644 --- a/graylog2-server/src/test/java/org/graylog2/events/ClusterEventCleanupPeriodicalTest.java +++ b/graylog2-server/src/test/java/org/graylog2/events/ClusterEventCleanupPeriodicalTest.java @@ -22,6 +22,7 @@ import com.mongodb.DBObject; import org.graylog.testing.mongodb.MongoDBInstance; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoCollections; import org.graylog2.database.MongoConnection; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; import org.joda.time.DateTime; @@ -56,8 +57,9 @@ public void setUpService() throws Exception { this.mongoConnection = mongodb.mongoConnection(); - MongoJackObjectMapperProvider provider = new MongoJackObjectMapperProvider(objectMapper); - this.clusterEventCleanupPeriodical = new ClusterEventCleanupPeriodical(provider, mongodb.mongoConnection()); + this.clusterEventCleanupPeriodical = new ClusterEventCleanupPeriodical(new MongoCollections( + new MongoJackObjectMapperProvider(objectMapper), + mongodb.mongoConnection())); } @After diff --git a/graylog2-server/src/test/java/org/graylog2/events/ClusterEventPeriodicalTest.java b/graylog2-server/src/test/java/org/graylog2/events/ClusterEventPeriodicalTest.java index b84f42dee6bf..cd04bf6941d0 100644 --- a/graylog2-server/src/test/java/org/graylog2/events/ClusterEventPeriodicalTest.java +++ b/graylog2-server/src/test/java/org/graylog2/events/ClusterEventPeriodicalTest.java @@ -77,17 +77,17 @@ public class ClusterEventPeriodicalTest { private ClusterEventBus clusterEventBus; private MongoConnection mongoConnection; private ClusterEventPeriodical clusterEventPeriodical; + private MongoJackObjectMapperProvider objectMapperProvider; @Before public void setUpService() throws Exception { DateTimeUtils.setCurrentMillisFixed(TIME.getMillis()); this.mongoConnection = mongodb.mongoConnection(); - - MongoJackObjectMapperProvider provider = new MongoJackObjectMapperProvider(objectMapper); + this.objectMapperProvider = new MongoJackObjectMapperProvider(objectMapper); this.clusterEventPeriodical = new ClusterEventPeriodical( - provider, + objectMapperProvider, mongodb.mongoConnection(), nodeId, new RestrictedChainingClassLoader(new ChainingClassLoader(getClass().getClassLoader()), @@ -287,9 +287,9 @@ public void prepareCollectionCreatesIndexesOnExistingCollection() throws Excepti assertThat(original.getName()).isEqualTo(ClusterEventPeriodical.COLLECTION_NAME); assertThat(original.getIndexInfo()).hasSize(1); - DBCollection collection = ClusterEventPeriodical.prepareCollection(mongoConnection); - assertThat(collection.getName()).isEqualTo(ClusterEventPeriodical.COLLECTION_NAME); - assertThat(collection.getIndexInfo()).hasSize(2); + final var collection = ClusterEventPeriodical.prepareCollection(mongoConnection, objectMapperProvider); + assertThat(collection.getNamespace().getCollectionName()).isEqualTo(ClusterEventPeriodical.COLLECTION_NAME); + assertThat(collection.listIndexes()).hasSize(2); assertThat(collection.getWriteConcern()).isEqualTo(WriteConcern.JOURNALED); } @@ -299,10 +299,10 @@ public void prepareCollectionCreatesCollectionIfItDoesNotExist() throws Exceptio final DB database = mongoConnection.getDatabase(); database.getCollection(ClusterEventPeriodical.COLLECTION_NAME).drop(); assertThat(database.collectionExists(ClusterEventPeriodical.COLLECTION_NAME)).isFalse(); - DBCollection collection = ClusterEventPeriodical.prepareCollection(mongoConnection); + final var collection = ClusterEventPeriodical.prepareCollection(mongoConnection, objectMapperProvider); - assertThat(collection.getName()).isEqualTo(ClusterEventPeriodical.COLLECTION_NAME); - assertThat(collection.getIndexInfo()).hasSize(2); + assertThat(collection.getNamespace().getCollectionName()).isEqualTo(ClusterEventPeriodical.COLLECTION_NAME); + assertThat(collection.listIndexes()).hasSize(2); assertThat(collection.getWriteConcern()).isEqualTo(WriteConcern.JOURNALED); } diff --git a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/EncodingTest.java b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/EncodingTest.java index ca59edb52928..bd6d40756249 100644 --- a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/EncodingTest.java +++ b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/EncodingTest.java @@ -22,6 +22,7 @@ import org.graylog2.plugin.MessageFactory; import org.graylog2.plugin.configuration.Configuration; import org.graylog2.plugin.inputs.codecs.Codec; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; import org.junit.jupiter.api.Assertions; @@ -55,20 +56,20 @@ class EncodingTest { void GelfCodecTestUTF8(MessageFactory messageFactory) { GelfCodec gelfCodecUTF8 = new GelfCodec(configUTF8, Mockito.mock(GelfChunkAggregator.class), messageFactory); - final Message message = gelfCodecUTF8.decode(rawUTF8); + final Message message = gelfCodecUTF8.decodeSafe(rawUTF8).get(); assertThat(message.getMessage()).isEqualTo(MESSAGE); - Assertions.assertThrows(IllegalStateException.class, () -> {gelfCodecUTF8.decode(rawUTF16);}); + Assertions.assertThrows(InputProcessingException.class, () -> gelfCodecUTF8.decodeSafe(rawUTF16).get()); } @Test void GelfCodecTestUTF16(MessageFactory messageFactory) { GelfCodec gelfCodecUTF16 = new GelfCodec(configUTF16, Mockito.mock(GelfChunkAggregator.class), messageFactory); - final Message message = gelfCodecUTF16.decode(rawUTF16); + final Message message = gelfCodecUTF16.decodeSafe(rawUTF16).get(); assertThat(message.getMessage()).isEqualTo(MESSAGE); - Assertions.assertThrows(IllegalStateException.class, () -> {gelfCodecUTF16.decode(rawUTF8);}); + Assertions.assertThrows(InputProcessingException.class, () -> gelfCodecUTF16.decodeSafe(rawUTF8).get()); } @Test @@ -78,10 +79,10 @@ void JsonPathCodecTestUTF8(MessageFactory messageFactory) { jsonPathCollectionUTF8.put(JsonPathCodec.CK_PATH, MSG_FIELD); JsonPathCodec jsonPathCodecUTF8 = new JsonPathCodec(new Configuration(jsonPathCollectionUTF8), objectMapperProvider.get(), messageFactory); - final Message message = jsonPathCodecUTF8.decode(rawUTF8); + final Message message = jsonPathCodecUTF8.decodeSafe(rawUTF8).get(); assertThat(message.getMessage()).contains(MESSAGE); - Assertions.assertThrows(PathNotFoundException.class, () -> {jsonPathCodecUTF8.decode(rawUTF16);}); + Assertions.assertThrows(PathNotFoundException.class, () -> jsonPathCodecUTF8.decodeSafe(rawUTF16).get()); } @Test @@ -91,9 +92,9 @@ void JsonPathCodecTestUTF16(MessageFactory messageFactory) { jsonPathCollectionUTF16.put(JsonPathCodec.CK_PATH, MSG_FIELD); JsonPathCodec jsonPathCodecUTF16 = new JsonPathCodec(new Configuration(jsonPathCollectionUTF16), objectMapperProvider.get(), messageFactory); - final Message message = jsonPathCodecUTF16.decode(rawUTF16); + final Message message = jsonPathCodecUTF16.decodeSafe(rawUTF16).get(); assertThat(message.getMessage()).contains(MESSAGE); - Assertions.assertThrows(PathNotFoundException.class, () -> {jsonPathCodecUTF16.decode(rawUTF8);}); + Assertions.assertThrows(PathNotFoundException.class, () -> jsonPathCodecUTF16.decodeSafe(rawUTF8).get()); } } diff --git a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java index 42d83f988c3d..e7787cf9a6d0 100644 --- a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java @@ -22,31 +22,25 @@ import org.graylog2.plugin.MessageFactory; import org.graylog2.plugin.TestMessageFactory; import org.graylog2.plugin.configuration.Configuration; +import org.graylog2.plugin.inputs.failure.InputProcessingException; import org.graylog2.plugin.journal.RawMessage; import org.joda.time.DateTime; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; +import org.mockito.junit.jupiter.MockitoExtension; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; -import static org.hamcrest.Matchers.isA; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.Assume.assumeTrue; +@ExtendWith(MockitoExtension.class) public class GelfCodecTest { - @Rule - public final MockitoRule mockitoRule = MockitoJUnit.rule(); - - @Rule - public final ExpectedException expectedException = ExpectedException.none(); @Mock private GelfChunkAggregator aggregator; @@ -61,14 +55,14 @@ public void setUp() { } @Test(expected = IllegalStateException.class) - public void decodeDoesNotThrowIllegalArgumentExceptionIfJsonIsInvalid() throws Exception { + public void decodeDoesNotThrowIllegalArgumentExceptionIfJsonIsInvalid() { // this fails gelf parsing, but empty Payloads are now ok. final RawMessage rawMessage = new RawMessage(new byte[0]); - codec.decode(rawMessage); + codec.decodeSafe(rawMessage); } @Test - public void decodeFiltersOutVersionField() throws Exception { + public void decodeFiltersOutVersionField() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -76,7 +70,7 @@ public void decodeFiltersOutVersionField() throws Exception { + "}"; final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertThat(message).isNotNull(); assertThat(message.getField("version")).isNull(); @@ -84,7 +78,7 @@ public void decodeFiltersOutVersionField() throws Exception { } @Test - public void decodeAllowsSettingCustomVersionField() throws Exception { + public void decodeAllowsSettingCustomVersionField() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -93,7 +87,7 @@ public void decodeAllowsSettingCustomVersionField() throws Exception { + "}"; final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertThat(message).isNotNull(); assertThat(message.getField("version")).isEqualTo("3.11"); @@ -101,7 +95,7 @@ public void decodeAllowsSettingCustomVersionField() throws Exception { } @Test - public void decodeBuildsValidMessageObject() throws Exception { + public void decodeBuildsValidMessageObject() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -115,7 +109,7 @@ public void decodeBuildsValidMessageObject() throws Exception { + "}"; final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertThat(message).isNotNull(); assertThat(message.getField("source")).isEqualTo("example.org"); @@ -125,51 +119,51 @@ public void decodeBuildsValidMessageObject() throws Exception { "_id", "source", "message", "full_message", "timestamp", "level", "user_id", "some_info", "some_env_var"); } + @Test public void decodeLargeCompressedMessageFails() throws Exception { - expectedException.expect(IllegalStateException.class); - expectedException.expectMessage("JSON is null/could not be parsed (invalid JSON)"); - expectedException.expectCause(isA(JsonParseException.class)); - final Configuration configuration = new Configuration(Collections.singletonMap("decompress_size_limit", 100)); final GelfCodec codec = new GelfCodec(configuration, aggregator, messageFactory); final String json = "{" - + "\"version\": \"1.1\"," - + "\"host\": \"example.org\"," - + "\"short_message\": \"A short message that helps you identify what is going on\"," - + "\"full_message\": \"Backtrace here\\n\\nMore stuff\"," - + "\"timestamp\": 1385053862.3072," - + "\"level\": 1," - + "\"_some_bytes1\": \"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, \"," - + "\"_some_bytes2\": \"sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, \"," - + "\"_some_bytes2\": \"sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum.\"" - + "}"; + + "\"version\": \"1.1\"," + + "\"host\": \"example.org\"," + + "\"short_message\": \"A short message that helps you identify what is going on\"," + + "\"full_message\": \"Backtrace here\\n\\nMore stuff\"," + + "\"timestamp\": 1385053862.3072," + + "\"level\": 1," + + "\"_some_bytes1\": \"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, \"," + + "\"_some_bytes2\": \"sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, \"," + + "\"_some_bytes2\": \"sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum.\"" + + "}"; final byte[] payload = TestHelper.zlibCompress(json); assumeTrue(payload.length > 100); final RawMessage rawMessage = new RawMessage(payload); - codec.decode(rawMessage); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasCauseInstanceOf(JsonParseException.class) + .hasMessage("JSON is null/could not be parsed (invalid JSON)"); } @Test - public void getAggregatorReturnsGelfChunkAggregator() throws Exception { + public void getAggregatorReturnsGelfChunkAggregator() { assertThat(codec.getAggregator()).isSameAs(aggregator); } @Test - public void decodeSucceedsWithoutHost() throws Exception { + public void decodeSucceedsWithoutHost() { final String json = "{" + "\"version\": \"1.1\"," + "\"short_message\": \"A short message that helps you identify what is going on\"" + "}"; final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertThat(message).isNotNull(); } @Test - public void decodeFailsWithWrongTypeForHost() throws Exception { + public void decodeFailsWithWrongTypeForHost() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": 42," @@ -178,13 +172,13 @@ public void decodeFailsWithWrongTypeForHost() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has invalid \"host\": 42"); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has invalid \"host\": 42"); } @Test - public void decodeFailsWithEmptyHost() throws Exception { + public void decodeFailsWithEmptyHost() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"\"," @@ -193,13 +187,13 @@ public void decodeFailsWithEmptyHost() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"host\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"host\" field."); } @Test - public void decodeFailsWithBlankHost() throws Exception { + public void decodeFailsWithBlankHost() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \" \"," @@ -208,13 +202,13 @@ public void decodeFailsWithBlankHost() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"host\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"host\" field."); } @Test - public void decodeFailsWithoutShortMessage() throws Exception { + public void decodeFailsWithoutShortMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"" @@ -222,13 +216,13 @@ public void decodeFailsWithoutShortMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> is missing mandatory \"short_message\" or \"message\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> is missing mandatory \"short_message\" or \"message\" field."); } @Test - public void decodeSucceedsWithoutShortMessageButWithMessage() throws Exception { + public void decodeSucceedsWithoutShortMessageButWithMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -237,12 +231,11 @@ public void decodeSucceedsWithoutShortMessageButWithMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); - assertThat(message).isNotNull(); + assertThat(codec.decodeSafe(rawMessage)).isNotEmpty(); } @Test - public void decodeSucceedsWithEmptyShortMessageButWithMessage() throws Exception { + public void decodeSucceedsWithEmptyShortMessageButWithMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -252,12 +245,11 @@ public void decodeSucceedsWithEmptyShortMessageButWithMessage() throws Exception final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); - assertThat(message).isNotNull(); + assertThat(codec.decodeSafe(rawMessage)).isNotEmpty(); } @Test - public void decodeFailsWithWrongTypeForShortMessage() throws Exception { + public void decodeFailsWithWrongTypeForShortMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -266,13 +258,13 @@ public void decodeFailsWithWrongTypeForShortMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has invalid \"short_message\": 42"); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has invalid \"short_message\": 42"); } @Test - public void decodeFailsWithWrongTypeForMessage() throws Exception { + public void decodeFailsWithWrongTypeForMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -281,13 +273,13 @@ public void decodeFailsWithWrongTypeForMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has invalid \"message\": 42"); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has invalid \"message\": 42"); } @Test - public void decodeFailsWithEmptyShortMessage() throws Exception { + public void decodeFailsWithEmptyShortMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -296,13 +288,13 @@ public void decodeFailsWithEmptyShortMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"short_message\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"short_message\" field."); } @Test - public void decodeFailsWithEmptyMessage() throws Exception { + public void decodeFailsWithEmptyMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -311,13 +303,13 @@ public void decodeFailsWithEmptyMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"message\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"message\" field."); } @Test - public void decodeFailsWithBlankShortMessage() throws Exception { + public void decodeFailsWithBlankShortMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -326,13 +318,13 @@ public void decodeFailsWithBlankShortMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"short_message\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"short_message\" field."); } @Test - public void decodeFailsWithBlankMessage() throws Exception { + public void decodeFailsWithBlankMessage() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -341,13 +333,13 @@ public void decodeFailsWithBlankMessage() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"message\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> has empty mandatory \"message\" field."); } @Test - public void decodeSucceedsWithWrongTypeForTimestamp() throws Exception { + public void decodeSucceedsWithWrongTypeForTimestamp() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"," @@ -361,7 +353,7 @@ public void decodeSucceedsWithWrongTypeForTimestamp() throws Exception { } @Test - public void decodeIncludesSourceAddressIfItFails() throws Exception { + public void decodeIncludesSourceAddressIfItFails() { final String json = "{" + "\"version\": \"1.1\"," + "\"host\": \"example.org\"" @@ -369,25 +361,25 @@ public void decodeIncludesSourceAddressIfItFails() throws Exception { final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8), new InetSocketAddress("198.51.100.42", 24783)); - assertThatIllegalArgumentException().isThrownBy(() -> codec.decode(rawMessage)) - .withNoCause() - .withMessageMatching("GELF message <[0-9a-f-]+> \\(received from <198\\.51\\.100\\.42:24783>\\) is missing mandatory \"short_message\" or \"message\" field."); + assertThatThrownBy(() -> codec.decodeSafe(rawMessage)) + .isInstanceOf(InputProcessingException.class) + .hasMessageMatching("GELF message <[0-9a-f-]+> \\(received from <198\\.51\\.100\\.42:24783>\\) is missing mandatory \"short_message\" or \"message\" field."); } @Test - public void decodeSucceedsWithMinimalMessages() throws Exception { - assertThat(codec.decode(new RawMessage("{\"short_message\":\"0\"}".getBytes(StandardCharsets.UTF_8)))).isNotNull(); - assertThat(codec.decode(new RawMessage("{\"message\":\"0\"}".getBytes(StandardCharsets.UTF_8)))).isNotNull(); + public void decodeSucceedsWithMinimalMessages() { + assertThat(codec.decodeSafe(new RawMessage("{\"short_message\":\"0\"}".getBytes(StandardCharsets.UTF_8)))).isNotEmpty(); + assertThat(codec.decodeSafe(new RawMessage("{\"message\":\"0\"}".getBytes(StandardCharsets.UTF_8)))).isNotEmpty(); } @Test - public void decodeSucceedsWithTrailingComma() throws Exception { - assertThat(codec.decode(new RawMessage("{\"short_message\":\"0\",}".getBytes(StandardCharsets.UTF_8)))).isNotNull(); - assertThat(codec.decode(new RawMessage("{\"message\":\"0\",}".getBytes(StandardCharsets.UTF_8)))).isNotNull(); + public void decodeSucceedsWithTrailingComma() { + assertThat(codec.decodeSafe(new RawMessage("{\"short_message\":\"0\",}".getBytes(StandardCharsets.UTF_8)))).isNotEmpty(); + assertThat(codec.decodeSafe(new RawMessage("{\"message\":\"0\",}".getBytes(StandardCharsets.UTF_8)))).isNotEmpty(); } @Test - public void decodeSucceedsWithValidTimestampIssue4027() throws Exception { + public void decodeSucceedsWithValidTimestampIssue4027() { // https://github.com/Graylog2/graylog2-server/issues/4027 final String json = "{" + "\"version\": \"1.1\"," @@ -397,13 +389,13 @@ public void decodeSucceedsWithValidTimestampIssue4027() throws Exception { + "}"; final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertThat(message).isNotNull(); assertThat(message.getTimestamp()).isEqualTo(DateTime.parse("2017-07-21T14:23:00.661Z")); } @Test - public void decodeSucceedsWithValidTimestampAsStringIssue4027() throws Exception { + public void decodeSucceedsWithValidTimestampAsStringIssue4027() { // https://github.com/Graylog2/graylog2-server/issues/4027 final String json = "{" + "\"version\": \"1.1\"," @@ -413,7 +405,7 @@ public void decodeSucceedsWithValidTimestampAsStringIssue4027() throws Exception + "}"; final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertThat(message).isNotNull(); assertThat(message.getTimestamp()).isEqualTo(DateTime.parse("2017-07-21T14:23:00.661Z")); } diff --git a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/JsonPathCodecTest.java b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/JsonPathCodecTest.java index 95721c026f03..69bd247d815d 100644 --- a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/JsonPathCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/JsonPathCodecTest.java @@ -50,7 +50,7 @@ private static Configuration configOf(String key1, Object value1, String key2, O } @Test - public void testReadResultingInSingleInteger() throws Exception { + public void testReadResultingInSingleInteger() { String json = "{\"url\":\"https://api.github.com/repos/Graylog2/graylog2-server/releases/assets/22660\",\"download_count\":76185,\"id\":22660,\"name\":\"graylog2-server-0.20.0-preview.1.tgz\",\"label\":\"graylog2-server-0.20.0-preview.1.tgz\",\"content_type\":\"application/octet-stream\",\"state\":\"uploaded\",\"size\":38179285,\"updated_at\":\"2013-09-30T20:05:46Z\"}"; String path = "$.download_count"; @@ -60,16 +60,16 @@ public void testReadResultingInSingleInteger() throws Exception { } @Test - public void testReadResultingInSingleIntegerFullJson() throws Exception { + public void testReadResultingInSingleIntegerFullJson() { RawMessage json = new RawMessage("{\"download_count\":76185}".getBytes(StandardCharsets.UTF_8)); String path = "$.download_count"; - Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decode(json); + Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decodeSafe(json).get(); assertThat(result.getField("download_count")).isEqualTo(76185); } @Test - public void testReadResultingInSingleString() throws Exception { + public void testReadResultingInSingleString() { String json = "{\"url\":\"https://api.github.com/repos/Graylog2/graylog2-server/releases/assets/22660\",\"download_count\":76185,\"id\":22660,\"name\":\"graylog2-server-0.20.0-preview.1.tgz\",\"label\":\"graylog2-server-0.20.0-preview.1.tgz\",\"content_type\":\"application/octet-stream\",\"state\":\"uploaded\",\"size\":38179285,\"updated_at\":\"2013-09-30T20:05:46Z\"}"; String path = "$.state"; @@ -79,16 +79,16 @@ public void testReadResultingInSingleString() throws Exception { } @Test - public void testReadResultingInSingleStringFullJson() throws Exception { + public void testReadResultingInSingleStringFullJson() { RawMessage json = new RawMessage("{\"url\":\"https://api.github.com/repos/Graylog2/graylog2-server/releases/assets/22660\",\"download_count\":76185,\"id\":22660,\"name\":\"graylog2-server-0.20.0-preview.1.tgz\",\"label\":\"graylog2-server-0.20.0-preview.1.tgz\",\"content_type\":\"application/octet-stream\",\"state\":\"uploaded\",\"size\":38179285,\"updated_at\":\"2013-09-30T20:05:46Z\"}".getBytes(StandardCharsets.UTF_8)); String path = "$.state"; - Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decode(json); + Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decodeSafe(json).get(); assertThat(result.getField("state")).isEqualTo("\"uploaded\""); } @Test - public void testReadFromMap() throws Exception { + public void testReadFromMap() { String json = "{\"store\":{\"book\":[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99,\"isbn\":\"0-553-21311-3\"}],\"bicycle\":{\"color\":\"red\",\"price\":19.95}}}"; String path = "$.store.book[?(@.category == 'fiction')].author"; @@ -98,16 +98,16 @@ public void testReadFromMap() throws Exception { } @Test - public void testReadFromMapFullJson() throws Exception { + public void testReadFromMapFullJson() { RawMessage json = new RawMessage("{\"store\":{\"book\":[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99,\"isbn\":\"0-553-21311-3\"}],\"bicycle\":{\"color\":\"red\",\"price\":19.95}}}".getBytes(StandardCharsets.UTF_8)); String path = "$.store.book[?(@.category == 'fiction')].author"; - Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decode(json); + Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decodeSafe(json).get(); assertThat(result.getField("store.book1.author")).isEqualTo("\"Evelyn Waugh\""); } @Test - public void testReadResultingInDouble() throws Exception { + public void testReadResultingInDouble() { String json = "{\"url\":\"https://api.github.com/repos/Graylog2/graylog2-server/releases/assets/22660\",\"some_double\":0.50,\"id\":22660,\"name\":\"graylog2-server-0.20.0-preview.1.tgz\",\"label\":\"graylog2-server-0.20.0-preview.1.tgz\",\"content_type\":\"application/octet-stream\",\"state\":\"uploaded\",\"size\":38179285,\"updated_at\":\"2013-09-30T20:05:46Z\"}"; String path = "$.some_double"; @@ -117,16 +117,16 @@ public void testReadResultingInDouble() throws Exception { } @Test - public void testReadResultingInDoubleFullJson() throws Exception { + public void testReadResultingInDoubleFullJson() { RawMessage json = new RawMessage("{\"url\":\"https://api.github.com/repos/Graylog2/graylog2-server/releases/assets/22660\",\"some_double\":0.50,\"id\":22660,\"name\":\"graylog2-server-0.20.0-preview.1.tgz\",\"label\":\"graylog2-server-0.20.0-preview.1.tgz\",\"content_type\":\"application/octet-stream\",\"state\":\"uploaded\",\"size\":38179285,\"updated_at\":\"2013-09-30T20:05:46Z\"}".getBytes(StandardCharsets.UTF_8)); String path = "$.store.book[?(@.category == 'fiction')].author"; - Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decode(json); + Message result = new JsonPathCodec(configOf(CK_PATH, path, CK_FLATTEN, true), objectMapperProvider.get(), messageFactory).decodeSafe(json).get(); assertThat(result.getField("some_double")).isEqualTo(0.5); } @Test - public void testBuildShortMessage() throws Exception { + public void testBuildShortMessage() { Map fields = Maps.newLinkedHashMap(); fields.put("baz", 9001); fields.put("foo", "bar"); @@ -136,7 +136,7 @@ public void testBuildShortMessage() throws Exception { } @Test - public void testBuildShortMessageFullJson() throws Exception { + public void testBuildShortMessageFullJson() { Map fields = Maps.newLinkedHashMap(); fields.put("baz", 9001); fields.put("foo", "bar"); @@ -146,7 +146,7 @@ public void testBuildShortMessageFullJson() throws Exception { } @Test - public void testBuildShortMessageThatGetsCut() throws Exception { + public void testBuildShortMessageThatGetsCut() { Map fields = Maps.newLinkedHashMap(); fields.put("baz", 9001); fields.put("foo", "bargggdzrtdfgfdgldfsjgkfdlgjdflkjglfdjgljslfperitperoujglkdnfkndsbafdofhasdpfoöadjsFOO"); @@ -156,7 +156,7 @@ public void testBuildShortMessageThatGetsCut() throws Exception { } @Test - public void testBuildShortMessageThatGetsCutFullJson() throws Exception { + public void testBuildShortMessageThatGetsCutFullJson() { Map fields = Maps.newLinkedHashMap(); fields.put("baz", 9001); fields.put("foo", "bargggdzrtdfgfdgldfsjgkfdlgjdflkjglfdjgljslfperitperoujglkdnfkndsbafdofhasdpfoöadjsFOO"); diff --git a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/SyslogCodecTest.java b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/SyslogCodecTest.java index dfc5b22ec295..61157ffb2591 100644 --- a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/SyslogCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/SyslogCodecTest.java @@ -81,8 +81,8 @@ public void setUp() throws Exception { } @Test - public void testDecodeStructured() throws Exception { - final Message message = codec.decode(buildRawMessage(STRUCTURED)); + public void testDecodeStructured() { + final Message message = codec.decodeSafe(buildRawMessage(STRUCTURED)).get(); assertNotNull(message); assertEquals("BOMAn application event log entry", message.getMessage()); @@ -98,8 +98,8 @@ public void testDecodeStructured() throws Exception { } @Test - public void testDecodeStructuredIssue845() throws Exception { - final Message message = codec.decode(buildRawMessage(STRUCTURED_ISSUE_845)); + public void testDecodeStructuredIssue845() { + final Message message = codec.decodeSafe(buildRawMessage(STRUCTURED_ISSUE_845)).get(); assertNotNull(message); assertEquals("User page 13 requested", message.getMessage()); @@ -117,11 +117,11 @@ public void testDecodeStructuredIssue845() throws Exception { } @Test - public void testDecodeStructuredIssue845WithExpandStructuredData() throws Exception { + public void testDecodeStructuredIssue845WithExpandStructuredData() { when(configuration.getBoolean(SyslogCodec.CK_EXPAND_STRUCTURED_DATA)).thenReturn(true); final SyslogCodec codec = new SyslogCodec(configuration, metricRegistry, messageFactory); - final Message message = codec.decode(buildRawMessage(STRUCTURED_ISSUE_845)); + final Message message = codec.decodeSafe(buildRawMessage(STRUCTURED_ISSUE_845)).get(); assertNotNull(message); assertEquals("User page 13 requested", message.getMessage()); @@ -139,8 +139,8 @@ public void testDecodeStructuredIssue845WithExpandStructuredData() throws Except } @Test - public void testDecodeStructuredIssue845Empty() throws Exception { - final Message message = codec.decode(buildRawMessage(STRUCTURED_ISSUE_845_EMPTY)); + public void testDecodeStructuredIssue845Empty() { + final Message message = codec.decodeSafe(buildRawMessage(STRUCTURED_ISSUE_845_EMPTY)).get(); assertNotNull(message); assertEquals("tralala", message.getMessage()); @@ -152,10 +152,10 @@ public void testDecodeStructuredIssue845Empty() throws Exception { } @Test - public void testDecodeStructuredWithFullMessage() throws Exception { + public void testDecodeStructuredWithFullMessage() { when(configuration.getBoolean(SyslogCodec.CK_STORE_FULL_MESSAGE)).thenReturn(true); - final Message message = codec.decode(buildRawMessage(STRUCTURED)); + final Message message = codec.decodeSafe(buildRawMessage(STRUCTURED)).get(); assertNotNull(message); assertEquals("BOMAn application event log entry", message.getMessage()); @@ -172,8 +172,8 @@ public void testDecodeStructuredWithFullMessage() throws Exception { } @Test - public void testDecodeStructuredIssue549() throws Exception { - final Message message = codec.decode(buildRawMessage(STRUCTURED_ISSUE_549)); + public void testDecodeStructuredIssue549() { + final Message message = codec.decodeSafe(buildRawMessage(STRUCTURED_ISSUE_549)).get(); assertNotNull(message); assertEquals("RT_FLOW_SESSION_DENY [junos@2636.1.1.1.2.39 source-address=\"1.2.3.4\" source-port=\"56639\" destination-address=\"5.6.7.8\" destination-port=\"2003\" service-name=\"None\" protocol-id=\"6\" icmp-type=\"0\" policy-name=\"log-all-else\" source-zone-name=\"campus\" destination-zone-name=\"mngmt\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth6.0\" encrypted=\"No\"]", message.getMessage()); @@ -198,8 +198,8 @@ public void testDecodeStructuredIssue549() throws Exception { } @Test - public void testDecodeUnstructured() throws Exception { - final Message message = codec.decode(buildRawMessage(UNSTRUCTURED)); + public void testDecodeUnstructured() { + final Message message = codec.decodeSafe(buildRawMessage(UNSTRUCTURED)).get(); assertNotNull(message); assertEquals("c4dc57ba1ebb syslog-ng[7208]: syslog-ng starting up; version='3.5.3'", message.getMessage()); @@ -212,10 +212,10 @@ public void testDecodeUnstructured() throws Exception { } @Test - public void testDecodeUnstructuredWithFullMessage() throws Exception { + public void testDecodeUnstructuredWithFullMessage() { when(configuration.getBoolean(SyslogCodec.CK_STORE_FULL_MESSAGE)).thenReturn(true); - final Message message = codec.decode(buildRawMessage(UNSTRUCTURED)); + final Message message = codec.decodeSafe(buildRawMessage(UNSTRUCTURED)).get(); assertNotNull(message); assertEquals("c4dc57ba1ebb syslog-ng[7208]: syslog-ng starting up; version='3.5.3'", message.getMessage()); @@ -269,7 +269,7 @@ public void rfc3164_section5_4_messages() { ); for (Map.Entry> entry : rfc3164messages.entrySet()) { - final Message message = codec.decode(buildRawMessage(entry.getKey())); + final Message message = codec.decodeSafe(buildRawMessage(entry.getKey())).get(); assertThat(message).isNotNull(); assertThat(message.getFields()).containsAllEntriesOf(entry.getValue()); } @@ -329,17 +329,17 @@ public void rfc5424_section6_5_messages() { ); for (Map.Entry> entry : rfc3164messages.entrySet()) { - final Message message = codec.decode(buildRawMessage(entry.getKey())); + final Message message = codec.decodeSafe(buildRawMessage(entry.getKey())).get(); assertThat(message).isNotNull(); assertThat(message.getFields()).containsAllEntriesOf(entry.getValue()); } } @Test - public void testIssue2954() throws Exception { + public void testIssue2954() { // https://github.com/Graylog2/graylog2-server/issues/2954 final RawMessage rawMessage = buildRawMessage("<6>2016-10-12T14:10:18Z hostname testmsg[20]: Test"); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertNotNull(message); assertEquals("hostname testmsg[20]: Test", message.getMessage()); @@ -351,10 +351,10 @@ public void testIssue2954() throws Exception { } @Test - public void testIssue3502() throws Exception { + public void testIssue3502() { // https://github.com/Graylog2/graylog2-server/issues/3502 final RawMessage rawMessage = buildRawMessage("<6>0 2017-02-15T16:01:07.000+01:00 hostname test - - - test 4"); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertNotNull(message); assertEquals("test 4", message.getMessage()); @@ -420,7 +420,7 @@ public void testCiscoSyslogMessages() { .build(); for (Map.Entry> entry : messages.entrySet()) { - final Message message = codec.decode(buildRawMessage(entry.getKey())); + final Message message = codec.decodeSafe(buildRawMessage(entry.getKey())).get(); assertThat(message).isNotNull(); assertThat(message.getFields()).containsAllEntriesOf(entry.getValue()); } @@ -439,7 +439,7 @@ public void testFortiGateTrimLineBreaks() { private void doTestFortigate(String fortigateMessage) { final RawMessage rawMessage = buildRawMessage(fortigateMessage); - final Message message = codec.decode(rawMessage); + final Message message = codec.decodeSafe(rawMessage).get(); assertThat(message).isNotNull(); assertThat(message.getMessage()).isEqualTo("date=2017-03-06 time=12:53:10 devname=DEVICENAME devid=DEVICEID logid=0000000013 type=traffic subtype=forward level=notice vd=ALIAS srcip=IP srcport=45748 srcintf=\"IF\" dstip=IP dstport=443 dstintf=\"IF\" sessionid=1122686199 status=close policyid=77 dstcountry=\"COUNTRY\" srccountry=\"COUNTRY\" trandisp=dnat tranip=IP tranport=443 service=HTTPS proto=6 appid=41540 app=\"SSL_TLSv1.2\" appcat=\"Network.Service\" applist=\"ACLNAME\" appact=detected duration=1 sentbyte=2313 rcvdbyte=14883 sentpkt=19 rcvdpkt=19 utmaction=passthrough utmevent=app-ctrl attack=\"SSL\" hostname=\"HOSTNAME\""); @@ -457,9 +457,9 @@ public void testDefaultTimezoneConfig() { when(configuration.getString("timezone")).thenReturn("MST"); SyslogCodec codec = new SyslogCodec(configuration, metricRegistry, messageFactory); - final Message msgWithoutTimezone = codec.decode(buildRawMessage(UNSTRUCTURED)); - final Message msgWithUTCTimezone = codec.decode(buildRawMessage(STRUCTURED)); - final Message msgWithTimezoneOffset = codec.decode(buildRawMessage(STRUCTURED_ISSUE_845_EMPTY)); + final Message msgWithoutTimezone = codec.decodeSafe(buildRawMessage(UNSTRUCTURED)).get(); + final Message msgWithUTCTimezone = codec.decodeSafe(buildRawMessage(STRUCTURED)).get(); + final Message msgWithTimezoneOffset = codec.decodeSafe(buildRawMessage(STRUCTURED_ISSUE_845_EMPTY)).get(); assertEquals(new DateTime(YEAR + "-10-21T12:09:37", DateTimeZone.forID("MST")).toDate(), ((DateTime) msgWithoutTimezone.getField("timestamp")).toDate()); assertEquals(new DateTime("2012-12-25T22:14:15.003Z", DateTimeZone.UTC), ((DateTime) msgWithUTCTimezone.getField("timestamp")).withZone(DateTimeZone.UTC)); @@ -471,9 +471,9 @@ public void testDefaultTimezoneConfigNotConfiguredStillUsesSystemTime() { when(configuration.getString("timezone")).thenReturn(DropdownField.NOT_CONFIGURED); SyslogCodec codec = new SyslogCodec(configuration, metricRegistry, messageFactory); - final Message msgWithoutTimezone = codec.decode(buildRawMessage(UNSTRUCTURED)); - final Message msgWithUTCTimezone = codec.decode(buildRawMessage(STRUCTURED)); - final Message msgWithTimezoneOffset = codec.decode(buildRawMessage(STRUCTURED_ISSUE_845_EMPTY)); + final Message msgWithoutTimezone = codec.decodeSafe(buildRawMessage(UNSTRUCTURED)).get(); + final Message msgWithUTCTimezone = codec.decodeSafe(buildRawMessage(STRUCTURED)).get(); + final Message msgWithTimezoneOffset = codec.decodeSafe(buildRawMessage(STRUCTURED_ISSUE_845_EMPTY)).get(); assertEquals(new DateTime(YEAR + "-10-21T12:09:37").toDate(), ((DateTime) msgWithoutTimezone.getField("timestamp")).toDate()); assertEquals(new DateTime("2012-12-25T22:14:15.003Z"), ((DateTime) msgWithUTCTimezone.getField("timestamp")));