Skip to content

Commit

Permalink
Merge branch 'master' into issue-16899
Browse files Browse the repository at this point in the history
  • Loading branch information
ousmaneo authored Nov 14, 2023
2 parents ea8608b + a95c74c commit fc23ff1
Show file tree
Hide file tree
Showing 93 changed files with 3,894 additions and 1,119 deletions.
17 changes: 17 additions & 0 deletions .github/workflows/changelog.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
name: "Changelog Check"

on:
pull_request:
types:
- "opened"
- "synchronize"
- "reopened"
- "edited"

jobs:
test:
name: "Check Changelog presence"
runs-on: "ubuntu-latest"

steps:
- uses: "Graylog2/actions/changelog-check@main"
47 changes: 47 additions & 0 deletions .github/workflows/dispatch-pr-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
name: Request dispatched PR Build

on:
- pull_request

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
dispatchedPR:
name: Dispatch wait and check
runs-on: ubuntu-latest

steps:
- name: Build pull requests string
id: pr_string
shell: bash
env:
PR_BODY: "${{ github.event.pull_request.body }}"
run: |
# Add the PR that triggered this workflow
pulls=("${{ github.repository }}#${{ github.event.number }}")
# Add any PR dependencies to the list of pulls
pulls+=($(grep -P '^/(jenkins-pr-deps|jpd|prd)' <<< "$PR_BODY" | \
grep -ioP '(Graylog2/\S+?#|https?://github.com/Graylog2/\S+?/pull/)[0-9]+' || true))
echo "pr_string=$(tr ' ' ',' <<< "${pulls[@]}")" >> $GITHUB_OUTPUT
- name: dispatch job to graylog-project-internal
run: >
gh workflow run -R Graylog2/graylog-project-internal pr-build.yml --ref master
-f caller_repo=${{ github.repository }}
-f caller_pr_nr=${{ github.event.number }}
-f caller_base_branch=${{ github.base_ref || github.ref_name }}
-f caller_head_branch=${{ github.head_ref }}
-f head_sha=${{ github.event.pull_request.head.sha }}
-f pull_requests="${{ steps.pr_string.outputs.pr_string }}"
-f initial_actor="${{ github.actor }}/${{ github.triggering_actor }}"
env:
GITHUB_TOKEN: ${{ secrets.PAT_GRAYLOG_PROJECT_INTERNAL_WORKFLOW_RW }}

- name: Give dispatched build time to add its status
run: sleep 20
# This is mostly cosmetic. If this workflow finishes before the dispatched
# build creates the status on the PR, the build will show up green for a while.
5 changes: 5 additions & 0 deletions changelog/unreleased/issue-17204.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "fixed"
message = "Fix tooltip colors on Rule Editor."

issues = ["17204"]
pulls = ["17220"]
13 changes: 13 additions & 0 deletions changelog/unreleased/issue-6032.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
type = "added"
message = "Added 4 pipeline functions for general use that were previously only available to Illuminate."

issues = ["Graylog2/graylog-plugin-enterprise#6032"]
pulls = ["6112"]

details.user = """
The following pipeline functions were added:
- array_contains
- array_remove
- string_array_add
- lookup_all
"""
5 changes: 5 additions & 0 deletions changelog/unreleased/pr-17042.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "a"
message = "Including basic data node usage information in telemetry."

pulls = ["17042"]

5 changes: 5 additions & 0 deletions changelog/unreleased/pr-17070.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "added"
message = "Add index set field type management page"

issues = ["17160"]
pulls = ["17070"]
7 changes: 7 additions & 0 deletions changelog/unreleased/pr-17073.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
type = "fixed"
message = "Fix Content pack install modal having two superposed modal element"

issues = ["16834", "15752"]
pulls = ["17073"]


5 changes: 5 additions & 0 deletions changelog/unreleased/pr-17087.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "f"
message = "Exclude non-message streams from aggregation event searches."

issues = ["Graylog2/graylog-plugin-enterprise#6042"]
pulls = ["17087"]
6 changes: 6 additions & 0 deletions changelog/unreleased/pr-17218.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
type = "fixed"
message = "Show ignore_null configuration on lookup cache form."

issues = ["15200"]
pulls = ["17218"]

6 changes: 6 additions & 0 deletions changelog/unreleased/pr-17233.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
type = "fixed"
message = "Fix redirect after creating contentpack revision."

issues = ["17071"]
pulls = ["17233"]

7 changes: 2 additions & 5 deletions data-node/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.4</version>
<version>3.2.0</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
Expand All @@ -498,7 +498,7 @@

<plugins>
<plugin>
<groupId>com.googlecode.maven-download-plugin</groupId>
<groupId>org.graylog.repackaged</groupId>
<artifactId>download-maven-plugin</artifactId>
<version>${download-maven-plugin.version}</version>
<executions>
Expand Down Expand Up @@ -694,9 +694,6 @@
</goals>
</execution>
</executions>
<configuration>
<useLastCommittedRevision>true</useLastCommittedRevision>
</configuration>
</plugin>
<plugin>
<groupId>com.mycila</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import org.graylog.shaded.opensearch2.org.opensearch.client.indices.AnalyzeRequest;
import org.graylog.shaded.opensearch2.org.opensearch.client.indices.AnalyzeResponse;
import org.graylog.shaded.opensearch2.org.opensearch.common.xcontent.XContentType;
import org.graylog.shaded.opensearch2.org.opensearch.rest.RestStatus;
import org.graylog.shaded.opensearch2.org.opensearch.core.rest.RestStatus;
import org.graylog2.indexer.messages.ChunkedBulkIndexer;
import org.graylog2.indexer.messages.DocumentNotFoundException;
import org.graylog2.indexer.messages.Indexable;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.graylog.storage.opensearch2;

import org.graylog.shaded.opensearch2.org.opensearch.common.text.Text;
import org.graylog.shaded.opensearch2.org.opensearch.core.common.text.Text;
import org.graylog.shaded.opensearch2.org.opensearch.search.SearchHit;
import org.graylog.shaded.opensearch2.org.opensearch.search.fetch.subphase.highlight.HighlightField;
import org.graylog2.indexer.results.ResultMessage;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@
import org.graylog.plugins.views.search.filter.QueryStringFilter;
import org.graylog.plugins.views.search.filter.StreamFilter;
import org.graylog.plugins.views.search.searchfilters.db.UsedSearchFiltersToQueryStringsMapper;
import org.graylog.shaded.opensearch2.org.opensearch.action.ShardOperationFailedException;
import org.graylog.shaded.opensearch2.org.opensearch.action.search.MultiSearchResponse;
import org.graylog.shaded.opensearch2.org.opensearch.action.search.SearchRequest;
import org.graylog.shaded.opensearch2.org.opensearch.action.search.SearchResponse;
import org.graylog.shaded.opensearch2.org.opensearch.action.support.IndicesOptions;
import org.graylog.shaded.opensearch2.org.opensearch.core.action.ShardOperationFailedException;
import org.graylog.shaded.opensearch2.org.opensearch.index.query.BoolQueryBuilder;
import org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilder;
import org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilders;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.graylog.plugins.views.search.SearchType;
import org.graylog.plugins.views.search.searchtypes.MessageList;
import org.graylog.plugins.views.search.searchtypes.Sort;
import org.graylog.shaded.opensearch2.org.opensearch.common.text.Text;
import org.graylog.shaded.opensearch2.org.opensearch.core.common.text.Text;
import org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilders;
import org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryStringQueryBuilder;
import org.graylog.shaded.opensearch2.org.opensearch.search.SearchHit;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,12 @@
*/
package org.graylog.storage.opensearch2;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.graylog.shaded.opensearch2.org.opensearch.action.admin.indices.refresh.RefreshRequest;
import org.graylog.shaded.opensearch2.org.opensearch.action.index.IndexRequest;
import org.graylog.shaded.opensearch2.org.opensearch.action.index.IndexResponse;
import org.graylog.shaded.opensearch2.org.opensearch.client.core.CountRequest;
import org.graylog.shaded.opensearch2.org.opensearch.client.core.CountResponse;
import org.graylog.shaded.opensearch2.org.opensearch.rest.RestStatus;
import org.graylog.shaded.opensearch2.org.opensearch.core.rest.RestStatus;
import org.graylog.storage.opensearch2.testing.OpenSearchInstance;
import org.graylog.testing.elasticsearch.SearchServerInstance;
import org.graylog2.indexer.messages.MessagesIT;
import org.graylog2.shared.bindings.providers.ObjectMapperProvider;
import org.junit.Rule;

import java.util.Map;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.graylog.shaded.opensearch2.org.opensearch.client.Response;
import org.graylog.shaded.opensearch2.org.opensearch.client.ResponseException;
import org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient;
import org.graylog.shaded.opensearch2.org.opensearch.rest.RestStatus;
import org.graylog.shaded.opensearch2.org.opensearch.core.rest.RestStatus;
import org.graylog2.indexer.BatchSizeTooLargeException;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand Down
5 changes: 1 addition & 4 deletions graylog2-server/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.4</version>
<version>3.2.0</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
Expand Down Expand Up @@ -989,9 +989,6 @@
</goals>
</execution>
</executions>
<configuration>
<useLastCommittedRevision>true</useLastCommittedRevision>
</configuration>
</plugin>
<plugin>
<groupId>com.mycila</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ public Set<String> buildEventSourceStreams(Set<String> searchStreams, Set<String
// This can happen if the user didn't select any stream in the event definition and an event should be
// created based on the absence of a search result. (e.g. count() < 1)
// When the source streams field of an event is empty, every user can see it. That's why we need to add
// streams to the field. We decided to use all currently existing streams (minus the default event streams)
// streams to the field. We decided to use all currently existing streams (minus the non-message streams)
// to make sure only users that have access to all message streams can see the event.
sourceStreams = streamService.loadAll().stream()
.map(Persisted::getId)
.filter(streamId -> !Stream.DEFAULT_EVENT_STREAM_IDS.contains(streamId))
.filter(streamId -> !Stream.NON_MESSAGE_STREAM_IDS.contains(streamId))
.collect(Collectors.toSet());
} else if (searchStreams.isEmpty()) {
// If the search streams is empty, we search in all streams and so we include all source streams from the result.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString;
import org.graylog.plugins.views.search.errors.ParameterExpansionError;
import org.graylog.plugins.views.search.errors.SearchException;
import org.graylog.plugins.views.search.rest.PermittedStreams;
import org.graylog.plugins.views.search.searchtypes.pivot.HasField;
import org.graylog.plugins.views.search.searchtypes.pivot.SeriesSpec;
import org.graylog.plugins.views.search.searchtypes.pivot.series.HasOptionalField;
Expand All @@ -62,6 +63,7 @@
import javax.inject.Inject;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand Down Expand Up @@ -92,6 +94,7 @@ public interface Factory extends EventProcessor.Factory<AggregationEventProcesso
private final EventStreamService eventStreamService;
private final Messages messages;
private final NotificationService notificationService;
private final PermittedStreams permittedStreams;

@Inject
public AggregationEventProcessor(@Assisted EventDefinition eventDefinition,
Expand All @@ -100,7 +103,8 @@ public AggregationEventProcessor(@Assisted EventDefinition eventDefinition,
DBEventProcessorStateService stateService,
MoreSearch moreSearch,
EventStreamService eventStreamService,
Messages messages, NotificationService notificationService) {
Messages messages, NotificationService notificationService,
PermittedStreams permittedStreams) {
this.eventDefinition = eventDefinition;
this.config = (AggregationEventProcessorConfig) eventDefinition.config();
this.aggregationSearchFactory = aggregationSearchFactory;
Expand All @@ -110,6 +114,7 @@ public AggregationEventProcessor(@Assisted EventDefinition eventDefinition,
this.eventStreamService = eventStreamService;
this.messages = messages;
this.notificationService = notificationService;
this.permittedStreams = permittedStreams;
}

@Override
Expand Down Expand Up @@ -219,7 +224,10 @@ private Set<String> getStreams(AggregationEventProcessorParameters parameters) {

private void filterSearch(EventFactory eventFactory, AggregationEventProcessorParameters parameters,
EventConsumer<List<EventWithContext>> eventsConsumer) throws EventProcessorException {
final Set<String> streams = getStreams(parameters);
Set<String> streams = getStreams(parameters);
if (streams.isEmpty()) {
streams = new HashSet<>(permittedStreams.load(streamId -> true));
}

final AtomicInteger messageCount = new AtomicInteger(0);
final MoreSearch.ScrollCallback callback = (messages, continueScrolling) -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.MapBinder;
import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
import org.graylog.plugins.pipelineprocessor.functions.arrays.ArrayContains;
import org.graylog.plugins.pipelineprocessor.functions.arrays.ArrayRemove;
import org.graylog.plugins.pipelineprocessor.functions.arrays.StringArrayAdd;
import org.graylog.plugins.pipelineprocessor.functions.conversion.BooleanConversion;
import org.graylog.plugins.pipelineprocessor.functions.conversion.CsvMapConversion;
import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion;
Expand Down Expand Up @@ -84,6 +87,7 @@
import org.graylog.plugins.pipelineprocessor.functions.lookup.ListGet;
import org.graylog.plugins.pipelineprocessor.functions.lookup.Lookup;
import org.graylog.plugins.pipelineprocessor.functions.lookup.LookupAddStringList;
import org.graylog.plugins.pipelineprocessor.functions.lookup.LookupAll;
import org.graylog.plugins.pipelineprocessor.functions.lookup.LookupAssignTtl;
import org.graylog.plugins.pipelineprocessor.functions.lookup.LookupClearKey;
import org.graylog.plugins.pipelineprocessor.functions.lookup.LookupHasValue;
Expand Down Expand Up @@ -293,6 +297,7 @@ protected void configure() {
addMessageProcessorFunction(LookupAssignTtl.NAME, LookupAssignTtl.class);
addMessageProcessorFunction(ListGet.NAME, ListGet.class);
addMessageProcessorFunction(ListCount.NAME, ListCount.class);
addMessageProcessorFunction(LookupAll.NAME, LookupAll.class);

// Maps
addMessageProcessorFunction(MapRemove.NAME, MapRemove.class);
Expand All @@ -303,6 +308,11 @@ protected void configure() {
// Debug
addMessageProcessorFunction(Debug.NAME, Debug.class);
addMessageProcessorFunction(MetricCounterIncrement.NAME, MetricCounterIncrement.class);

// Arrays
addMessageProcessorFunction(ArrayContains.NAME, ArrayContains.class);
addMessageProcessorFunction(ArrayRemove.NAME, ArrayRemove.class);
addMessageProcessorFunction(StringArrayAdd.NAME, StringArrayAdd.class);
}

protected void addMessageProcessorFunction(String name, Class<? extends Function<?>> functionClass) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.plugins.pipelineprocessor.functions.arrays;

import com.google.common.collect.ImmutableList;
import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction;

import java.util.Collection;
import java.util.List;

abstract public class AbstractArrayFunction<T> extends AbstractFunction<T> {
@SuppressWarnings("rawtypes")
static List toList(Object obj) {
if (obj instanceof Collection) {
return ImmutableList.copyOf((Collection) obj);
} else {
throw new IllegalArgumentException("Unsupported data type for parameter 'elements': " + obj.getClass().getCanonicalName());
}
}
}
Loading

0 comments on commit fc23ff1

Please sign in to comment.