Merge branch 'master' into datanode-migration-ui

UPGRADING.md

@@ -2,7 +2,7 @@ Upgrading to Graylog 6.0.x
 ==========================
 
 ## Breaking Changes
-
+- Default value for `data_dir` configuration option has been removed and must be specified in `graylog.conf`.
 ### Changed default number of process-buffer and output-buffer processors
 
 The default values for the configuration settings `processbuffer_processors` and `outputbuffer_processors` have been

changelog/unreleased/issue-13673.toml

@@ -0,0 +1,5 @@
+type = "c"
+message = "The default value for the `data_dir` configuration option has been removed. It needs to be configured explicitly now."
+
+issues = ["13673"]
+pulls = ["15902"]

changelog/unreleased/issue-7386.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Add search result statistics on the main search screen and execution info popover to each dashboard widget"
+
+pulls = ["16130"]
+issues=["7386"]

data-node/src/main/java/org/graylog/datanode/bootstrap/CmdLineTool.java

@@ -25,7 +25,6 @@
 import com.github.joschi.jadconfig.RepositoryException;
 import com.github.joschi.jadconfig.ValidationException;
 import com.github.joschi.jadconfig.guava.GuavaConverterFactory;
-import com.github.joschi.jadconfig.guice.NamedConfigParametersModule;
 import com.github.joschi.jadconfig.jodatime.JodaTimeConverterFactory;
 import com.github.joschi.jadconfig.repositories.EnvironmentRepository;
 import com.github.joschi.jadconfig.repositories.PropertiesRepository;
@@ -36,6 +35,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import com.google.inject.AbstractModule;
 import com.google.inject.Binder;
 import com.google.inject.CreationException;
 import com.google.inject.Guice;
@@ -51,6 +51,7 @@
 import org.apache.logging.log4j.core.LoggerContext;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.graylog.datanode.bootstrap.commands.MigrateCmd;
+import org.graylog2.bindings.NamedConfigParametersOverrideModule;
 import org.graylog2.bootstrap.CliCommand;
 import org.graylog2.configuration.PathConfiguration;
 import org.graylog2.configuration.TLSProtocolsConfiguration;
@@ -287,7 +288,7 @@ public void doRun(Level logLevel) {
 
         injector = setupInjector(
                 new IsDevelopmentBindings(),
-                new NamedConfigParametersModule(jadConfig.getConfigurationBeans()),
+                new NamedConfigParametersOverrideModule(jadConfig.getConfigurationBeans()),
                 binder -> binder.bind(MetricRegistry.class).toInstance(metricRegistry)
         );
 
@@ -453,8 +454,8 @@ protected Injector setupInjector(Module... modules) {
      * configuration values in modules at binding time.
      */
     protected Injector setupCoreConfigInjector() {
-        final NamedConfigParametersModule configModule =
-                new NamedConfigParametersModule(jadConfig.getConfigurationBeans());
+        final AbstractModule configModule =
+                new NamedConfigParametersOverrideModule(jadConfig.getConfigurationBeans());
 
         Injector coreConfigInjector = null;
         try {

data-node/src/main/java/org/graylog/datanode/bootstrap/ServerBootstrap.java

@@ -16,7 +16,6 @@
  */
 package org.graylog.datanode.bootstrap;
 
-import com.github.joschi.jadconfig.guice.NamedConfigParametersModule;
 import com.github.rvesse.airline.annotations.Option;
 import com.google.common.util.concurrent.ServiceManager;
 import com.google.inject.Binder;
@@ -31,6 +30,7 @@
 import org.graylog.datanode.bindings.GenericInitializerBindings;
 import org.graylog.datanode.bindings.PreflightChecksBindings;
 import org.graylog.datanode.bindings.SchedulerBindings;
+import org.graylog2.bindings.NamedConfigParametersOverrideModule;
 import org.graylog2.bootstrap.preflight.MongoDBPreflightCheck;
 import org.graylog2.bootstrap.preflight.PreflightCheckException;
 import org.graylog2.bootstrap.preflight.PreflightCheckService;
@@ -124,7 +124,7 @@ private void runPreFlightChecks(Set<Plugin> plugins) {
             return;
         }
 
-       runMongoPreflightCheck();
+        runMongoPreflightCheck();
 
         final List<Module> preflightCheckModules = plugins.stream().map(Plugin::preflightCheckModules)
                 .flatMap(Collection::stream).collect(Collectors.toList());
@@ -153,7 +153,7 @@ private void runMongoPreflightCheck() {
     private Injector getMongoPreFlightInjector() {
         return Guice.createInjector(
                 new IsDevelopmentBindings(),
-                new NamedConfigParametersModule(jadConfig.getConfigurationBeans()),
+                new NamedConfigParametersOverrideModule(jadConfig.getConfigurationBeans()),
                 new ConfigurationModule(configuration),
                 new DatanodeConfigurationBindings()
 
@@ -163,7 +163,7 @@ private Injector getMongoPreFlightInjector() {
     private Injector getPreflightInjector(List<Module> preflightCheckModules) {
         return Guice.createInjector(
                 new IsDevelopmentBindings(),
-                new NamedConfigParametersModule(jadConfig.getConfigurationBeans()),
+                new NamedConfigParametersOverrideModule(jadConfig.getConfigurationBeans()),
                 new ConfigurationModule(configuration),
                 new PreflightChecksBindings(),
                 new DatanodeConfigurationBindings(),

data-node/src/test/java/org/graylog/datanode/testinfra/DatanodeDevContainerBuilder.java

@@ -139,7 +139,7 @@ public GenericContainer<?> build() {
         GenericContainer<?> container = new GenericContainer<>(imageSupplier.get())
                 .withExposedPorts(restPort, openSearchHttpPort)
                 .withNetwork(network)
-
+                .withEnv("GRAYLOG_DATANODE_DATA_DIR", "data")
                 .withEnv("GRAYLOG_DATANODE_OPENSEARCH_LOCATION", IMAGE_WORKING_DIR)
                 .withEnv("GRAYLOG_DATANODE_INSECURE_STARTUP", "true")
                 .withEnv("GRAYLOG_DATANODE_CONFIG_LOCATION", IMAGE_WORKING_DIR + "/config") // this is the datanode config dir for certs

graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpProcessorConfig.java

@@ -18,22 +18,25 @@
 
 import com.github.joschi.jadconfig.Parameter;
 import com.github.joschi.jadconfig.validators.PathReadableValidator;
+import org.graylog2.bindings.NamedBindingOverride;
 import org.graylog2.configuration.PathConfiguration;
 
 import jakarta.inject.Singleton;
 
 import java.nio.file.Path;
+import java.util.Optional;
 
 @Singleton
 public class GeoIpProcessorConfig extends PathConfiguration {
     private static final String PREFIX = "geo_ip_processor";
     public static final String S3_DOWNLOAD_LOCATION = PREFIX + "_s3_download_location";
 
-    @Parameter(value = S3_DOWNLOAD_LOCATION, required = true, validators = PathReadableValidator.class)
-    private final Path s3DownloadLocation = DEFAULT_DATA_DIR.resolve("geolocation");
+    @Parameter(value = S3_DOWNLOAD_LOCATION, validators = PathReadableValidator.class)
+    private Path s3DownloadLocation;
 
+    @NamedBindingOverride(value = S3_DOWNLOAD_LOCATION)
     public Path getS3DownloadLocation() {
-        return s3DownloadLocation;
+        return Optional.ofNullable(s3DownloadLocation).orElse(getDataDir().resolve("geolocation"));
     }
 
 }

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActions.java

@@ -16,8 +16,6 @@
  */
 package org.graylog.plugins.views.storage.migration.state.actions;
 
-import java.util.Map;
-
 /**
  * Set of callbacks used during the migration in different states.
  */
@@ -42,5 +40,11 @@ public interface MigrationActions extends WithArgs {
     boolean caDoesNotExist();
     boolean removalPolicyDoesNotExist();
     boolean caAndRemovalPolicyExist();
+
+    void provisionDataNodes();
+
+    boolean provisioningFinished();
+
     void resetMigration();
+
 }

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java

@@ -20,19 +20,30 @@
 import org.graylog.plugins.views.storage.migration.state.persistence.DatanodeMigrationConfiguration;
 import org.graylog.security.certutil.CaService;
 import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException;
+import org.graylog2.cluster.nodes.DataNodeDto;
+import org.graylog2.cluster.nodes.DataNodeStatus;
+import org.graylog2.cluster.nodes.NodeService;
+import org.graylog2.cluster.preflight.DataNodeProvisioningConfig;
+import org.graylog2.cluster.preflight.DataNodeProvisioningService;
 import org.graylog2.plugin.certificates.RenewalPolicy;
 import org.graylog2.plugin.cluster.ClusterConfigService;
 
+import java.util.Map;
+
 public class MigrationActionsImpl implements MigrationActions {
 
     private final ClusterConfigService clusterConfigService;
+    private final NodeService<DataNodeDto> nodeService;
     private final CaService caService;
+    private final DataNodeProvisioningService dataNodeProvisioningService;
 
     @Inject
-    public MigrationActionsImpl(final ClusterConfigService clusterConfigService,
-                                final CaService caService) {
+    public MigrationActionsImpl(final ClusterConfigService clusterConfigService, NodeService<DataNodeDto> nodeService,
+                                final CaService caService, DataNodeProvisioningService dataNodeProvisioningService) {
         this.clusterConfigService = clusterConfigService;
+        this.nodeService = nodeService;
         this.caService = caService;
+        this.dataNodeProvisioningService = dataNodeProvisioningService;
     }
 
     @Override
@@ -108,4 +119,15 @@ public boolean removalPolicyDoesNotExist() {
     public boolean caAndRemovalPolicyExist() {
         return !caDoesNotExist() && !removalPolicyDoesNotExist();
     }
+
+    @Override
+    public void provisionDataNodes() {
+        final Map<String, DataNodeDto> activeDataNodes = nodeService.allActive();
+        activeDataNodes.values().forEach(node -> dataNodeProvisioningService.changeState(node.getNodeId(), DataNodeProvisioningConfig.State.CONFIGURED));
+    }
+
+    @Override
+    public boolean provisioningFinished() {
+        return nodeService.allActive().values().stream().allMatch(node -> node.getDataNodeStatus() == DataNodeStatus.AVAILABLE);
+    }
 }

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/machine/MigrationState.java

@@ -26,6 +26,7 @@ public enum MigrationState {
     ROLLING_UPGRADE_MIGRATION_WELCOME_PAGE,
     REMOTE_REINDEX_WELCOME_PAGE,
     PROVISION_DATANODE_CERTIFICATES_PAGE,
+    PROVISION_DATANODE_CERTIFICATES_RUNNING,
     EXISTING_DATA_MIGRATION_QUESTION_PAGE,
     MIGRATE_EXISTING_DATA,
     ASK_TO_SHUTDOWN_OLD_CLUSTER,

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/machine/MigrationStateMachineBuilder.java

@@ -38,19 +38,18 @@ private static StateMachineConfig<MigrationState, MigrationStep> configureStates
                 .permit(MigrationStep.SELECT_MIGRATION, MigrationState.MIGRATION_WELCOME_PAGE, () -> LOG.info("Migration selected in menu, show welcome page"));
 
         config.configure(MigrationState.MIGRATION_WELCOME_PAGE)
-                .permit(MigrationStep.SHOW_DIRECTORY_COMPATIBILITY_CHECK, MigrationState.DIRECTORY_COMPATIBILITY_CHECK_PAGE, () -> LOG.info("Showing directory compatibility check page"));
+                .permit(MigrationStep.SHOW_DIRECTORY_COMPATIBILITY_CHECK, MigrationState.DIRECTORY_COMPATIBILITY_CHECK_PAGE, () -> LOG.info("Showing directory compatibility check page"))
+                .permit(MigrationStep.SKIP_DIRECTORY_COMPATIBILITY_CHECK, MigrationState.CA_CREATION_PAGE);
 
         config.configure(MigrationState.DIRECTORY_COMPATIBILITY_CHECK_PAGE)
                 .onEntry(migrationActions::runDirectoryCompatibilityCheck)
-                .permitIf(MigrationStep.SHOW_CA_CREATION, MigrationState.CA_CREATION_PAGE, migrationActions::directoryCompatibilityCheckOk)
-                .permitIf(MigrationStep.SHOW_RENEWAL_POLICY_CREATION, MigrationState.RENEWAL_POLICY_CREATION_PAGE, migrationActions::directoryCompatibilityCheckOk);
+                .permitIf(MigrationStep.SHOW_CA_CREATION, MigrationState.CA_CREATION_PAGE, migrationActions::directoryCompatibilityCheckOk);
 
         config.configure(MigrationState.CA_CREATION_PAGE)
-                .permitIf(MigrationStep.SHOW_RENEWAL_POLICY_CREATION, MigrationState.RENEWAL_POLICY_CREATION_PAGE, migrationActions::caDoesNotExist)
+                .permitIf(MigrationStep.SHOW_RENEWAL_POLICY_CREATION, MigrationState.RENEWAL_POLICY_CREATION_PAGE, () -> !migrationActions.caDoesNotExist())
                 .permitIf(MigrationStep.SHOW_MIGRATION_SELECTION, MigrationState.MIGRATION_SELECTION_PAGE, migrationActions::caAndRemovalPolicyExist);
 
         config.configure(MigrationState.RENEWAL_POLICY_CREATION_PAGE)
-                .permitIf(MigrationStep.SHOW_CA_CREATION, MigrationState.CA_CREATION_PAGE, migrationActions::removalPolicyDoesNotExist)
                 .permitIf(MigrationStep.SHOW_MIGRATION_SELECTION, MigrationState.MIGRATION_SELECTION_PAGE, migrationActions::caAndRemovalPolicyExist);
 
         // Major decision - remote reindexing or rolling upgrade(in-place)?
@@ -65,7 +64,10 @@ private static StateMachineConfig<MigrationState, MigrationStep> configureStates
                 });
 
         config.configure(MigrationState.PROVISION_DATANODE_CERTIFICATES_PAGE)
-                .permit(MigrationStep.SHOW_DATA_MIGRATION_QUESTION, MigrationState.EXISTING_DATA_MIGRATION_QUESTION_PAGE);
+                .permit(MigrationStep.PROVISION_DATANODE_CERTIFICATES, MigrationState.PROVISION_DATANODE_CERTIFICATES_RUNNING, migrationActions::provisionDataNodes);
+
+        config.configure(MigrationState.PROVISION_DATANODE_CERTIFICATES_RUNNING)
+                .permitIf(MigrationStep.SHOW_DATA_MIGRATION_QUESTION, MigrationState.EXISTING_DATA_MIGRATION_QUESTION_PAGE, migrationActions::provisioningFinished);
 
         config.configure(MigrationState.EXISTING_DATA_MIGRATION_QUESTION_PAGE)
                 .permit(MigrationStep.SHOW_MIGRATE_EXISTING_DATA, MigrationState.MIGRATE_EXISTING_DATA)
@@ -83,7 +85,7 @@ private static StateMachineConfig<MigrationState, MigrationStep> configureStates
                 .permit(MigrationStep.SHOW_PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES, MigrationState.PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES, migrationActions::directoryCompatibilityCheckOk);
 
         config.configure(MigrationState.PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES)
-                .permit(MigrationStep.CALCULATE_JOURNAL_SIZE, MigrationState.JOURNAL_SIZE_DOWNTIME_WARNING);
+                .permit(MigrationStep.CALCULATE_JOURNAL_SIZE, MigrationState.JOURNAL_SIZE_DOWNTIME_WARNING, migrationActions::provisionDataNodes);
 
         config.configure(MigrationState.JOURNAL_SIZE_DOWNTIME_WARNING)
                 .permit(MigrationStep.SHOW_STOP_PROCESSING_PAGE, MigrationState.MESSAGE_PROCESSING_STOP_REPLACE_CLUSTER_AND_MP_RESTART);

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/machine/MigrationStep.java

@@ -19,9 +19,11 @@
 public enum MigrationStep {
     SELECT_MIGRATION,
     SHOW_DIRECTORY_COMPATIBILITY_CHECK,
+    SKIP_DIRECTORY_COMPATIBILITY_CHECK,
     SHOW_CA_CREATION,
     SHOW_RENEWAL_POLICY_CREATION,
     SHOW_MIGRATION_SELECTION,
+    PROVISION_DATANODE_CERTIFICATES,
     SHOW_DATA_MIGRATION_QUESTION,
     SHOW_MIGRATE_EXISTING_DATA,
     SHOW_ASK_TO_SHUTDOWN_OLD_CLUSTER,
@@ -45,5 +47,4 @@ public enum MigrationStep {
     START_DATANODE_CLUSTER,
     START_MESSAGE_PROCESSING,
     CONFIRM_OLD_CONNECTION_STRING_FROM_CONFIG_REMOVED,
-
 }

graylog2-server/src/main/java/org/graylog2/Configuration.java

@@ -32,6 +32,7 @@
 import org.graylog.plugins.views.search.engine.suggestions.FieldValueSuggestionMode;
 import org.graylog.plugins.views.search.engine.suggestions.FieldValueSuggestionModeConverter;
 import org.graylog.security.certutil.CaConfiguration;
+import org.graylog2.bindings.NamedBindingOverride;
 import org.graylog2.cluster.leader.AutomaticLeaderElectionService;
 import org.graylog2.cluster.leader.LeaderElectionMode;
 import org.graylog2.cluster.leader.LeaderElectionService;
@@ -49,6 +50,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collections;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.graylog2.shared.utilities.StringUtils.f;
@@ -60,6 +62,7 @@
 public class Configuration extends CaConfiguration {
     public static final String SAFE_CLASSES = "safe_classes";
 
+    public static final String CONTENT_PACKS_DIR = "content_packs_dir";
     /**
      * Deprecated! Use isLeader() instead.
      */
@@ -168,8 +171,8 @@ public class Configuration extends CaConfiguration {
     @Parameter(value = "content_packs_loader_enabled")
     private boolean contentPacksLoaderEnabled = false;
 
-    @Parameter(value = "content_packs_dir")
-    private Path contentPacksDir = DEFAULT_DATA_DIR.resolve("contentpacks");
+    @Parameter(value = CONTENT_PACKS_DIR)
+    private Path contentPacksDir;
 
     @Parameter(value = "content_packs_auto_install", converter = TrimmedStringSetConverter.class)
     private Set<String> contentPacksAutoInstall = Collections.emptySet();
@@ -430,8 +433,9 @@ public boolean isContentPacksLoaderEnabled() {
         return contentPacksLoaderEnabled;
     }
 
+    @NamedBindingOverride(value = CONTENT_PACKS_DIR)
     public Path getContentPacksDir() {
-        return contentPacksDir;
+        return Optional.ofNullable(contentPacksDir).orElse(getDataDir().resolve("contentpacks"));
     }
 
     public Set<String> getContentPacksAutoInstall() {

graylog2-server/src/main/java/org/graylog2/bindings/NamedBindingOverride.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.bindings;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+/**
+ * Annotation to override named bindings of a configuration parameter in a configuration bean
+ * which is used in {@link NamedConfigParametersOverrideModule}
+ *
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface NamedBindingOverride {
+    String value();
+}