Add data node provisioning resource (#18178)

* add datanode provisioning rest resource

* add provisioning to migration actions and state machine transition

* add provisioning to rolling update

* add intermediary state for async provisioning

* add adapter method

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActions.java

@@ -16,8 +16,6 @@
  */
 package org.graylog.plugins.views.storage.migration.state.actions;
 
-import java.util.Map;
-
 /**
  * Set of callbacks used during the migration in different states.
  */
@@ -42,5 +40,11 @@ public interface MigrationActions extends WithArgs {
     boolean caDoesNotExist();
     boolean removalPolicyDoesNotExist();
     boolean caAndRemovalPolicyExist();
+
+    void provisionDataNodes();
+
+    boolean provisioningFinished();
+
     void resetMigration();
+
 }

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java

@@ -20,19 +20,30 @@
 import org.graylog.plugins.views.storage.migration.state.persistence.DatanodeMigrationConfiguration;
 import org.graylog.security.certutil.CaService;
 import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException;
+import org.graylog2.cluster.nodes.DataNodeDto;
+import org.graylog2.cluster.nodes.DataNodeStatus;
+import org.graylog2.cluster.nodes.NodeService;
+import org.graylog2.cluster.preflight.DataNodeProvisioningConfig;
+import org.graylog2.cluster.preflight.DataNodeProvisioningService;
 import org.graylog2.plugin.certificates.RenewalPolicy;
 import org.graylog2.plugin.cluster.ClusterConfigService;
 
+import java.util.Map;
+
 public class MigrationActionsImpl implements MigrationActions {
 
     private final ClusterConfigService clusterConfigService;
+    private final NodeService<DataNodeDto> nodeService;
     private final CaService caService;
+    private final DataNodeProvisioningService dataNodeProvisioningService;
 
     @Inject
-    public MigrationActionsImpl(final ClusterConfigService clusterConfigService,
-                                final CaService caService) {
+    public MigrationActionsImpl(final ClusterConfigService clusterConfigService, NodeService<DataNodeDto> nodeService,
+                                final CaService caService, DataNodeProvisioningService dataNodeProvisioningService) {
         this.clusterConfigService = clusterConfigService;
+        this.nodeService = nodeService;
         this.caService = caService;
+        this.dataNodeProvisioningService = dataNodeProvisioningService;
     }
 
     @Override
@@ -108,4 +119,15 @@ public boolean removalPolicyDoesNotExist() {
     public boolean caAndRemovalPolicyExist() {
         return !caDoesNotExist() && !removalPolicyDoesNotExist();
     }
+
+    @Override
+    public void provisionDataNodes() {
+        final Map<String, DataNodeDto> activeDataNodes = nodeService.allActive();
+        activeDataNodes.values().forEach(node -> dataNodeProvisioningService.changeState(node.getNodeId(), DataNodeProvisioningConfig.State.CONFIGURED));
+    }
+
+    @Override
+    public boolean provisioningFinished() {
+        return nodeService.allActive().values().stream().allMatch(node -> node.getDataNodeStatus() == DataNodeStatus.AVAILABLE);
+    }
 }

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/machine/MigrationState.java

@@ -26,6 +26,7 @@ public enum MigrationState {
     ROLLING_UPGRADE_MIGRATION_WELCOME_PAGE,
     REMOTE_REINDEX_WELCOME_PAGE,
     PROVISION_DATANODE_CERTIFICATES_PAGE,
+    PROVISION_DATANODE_CERTIFICATES_RUNNING,
     EXISTING_DATA_MIGRATION_QUESTION_PAGE,
     MIGRATE_EXISTING_DATA,
     ASK_TO_SHUTDOWN_OLD_CLUSTER,

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/machine/MigrationStateMachineBuilder.java

@@ -65,7 +65,10 @@ private static StateMachineConfig<MigrationState, MigrationStep> configureStates
                 });
 
         config.configure(MigrationState.PROVISION_DATANODE_CERTIFICATES_PAGE)
-                .permit(MigrationStep.SHOW_DATA_MIGRATION_QUESTION, MigrationState.EXISTING_DATA_MIGRATION_QUESTION_PAGE);
+                .permit(MigrationStep.PROVISION_DATANODE_CERTIFICATES, MigrationState.PROVISION_DATANODE_CERTIFICATES_RUNNING, migrationActions::provisionDataNodes);
+
+        config.configure(MigrationState.PROVISION_DATANODE_CERTIFICATES_RUNNING)
+                .permitIf(MigrationStep.SHOW_DATA_MIGRATION_QUESTION, MigrationState.EXISTING_DATA_MIGRATION_QUESTION_PAGE, migrationActions::provisioningFinished);
 
         config.configure(MigrationState.EXISTING_DATA_MIGRATION_QUESTION_PAGE)
                 .permit(MigrationStep.SHOW_MIGRATE_EXISTING_DATA, MigrationState.MIGRATE_EXISTING_DATA)
@@ -83,7 +86,7 @@ private static StateMachineConfig<MigrationState, MigrationStep> configureStates
                 .permit(MigrationStep.SHOW_PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES, MigrationState.PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES, migrationActions::directoryCompatibilityCheckOk);
 
         config.configure(MigrationState.PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES)
-                .permit(MigrationStep.CALCULATE_JOURNAL_SIZE, MigrationState.JOURNAL_SIZE_DOWNTIME_WARNING);
+                .permit(MigrationStep.CALCULATE_JOURNAL_SIZE, MigrationState.JOURNAL_SIZE_DOWNTIME_WARNING, migrationActions::provisionDataNodes);
 
         config.configure(MigrationState.JOURNAL_SIZE_DOWNTIME_WARNING)
                 .permit(MigrationStep.SHOW_STOP_PROCESSING_PAGE, MigrationState.MESSAGE_PROCESSING_STOP_REPLACE_CLUSTER_AND_MP_RESTART);

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/machine/MigrationStep.java

@@ -22,6 +22,7 @@ public enum MigrationStep {
     SHOW_CA_CREATION,
     SHOW_RENEWAL_POLICY_CREATION,
     SHOW_MIGRATION_SELECTION,
+    PROVISION_DATANODE_CERTIFICATES,
     SHOW_DATA_MIGRATION_QUESTION,
     SHOW_MIGRATE_EXISTING_DATA,
     SHOW_ASK_TO_SHUTDOWN_OLD_CLUSTER,
@@ -45,5 +46,4 @@ public enum MigrationStep {
     START_DATANODE_CLUSTER,
     START_MESSAGE_PROCESSING,
     CONFIRM_OLD_CONNECTION_STRING_FROM_CONFIG_REMOVED,
-
 }

graylog2-server/src/main/java/org/graylog2/bootstrap/preflight/web/resources/DataNodeProvisioningResource.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.bootstrap.preflight.web.resources;
+
+
+import io.swagger.annotations.Api;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import org.apache.shiro.authz.annotation.RequiresAuthentication;
+import org.graylog2.audit.jersey.NoAuditEvent;
+import org.graylog2.cluster.nodes.DataNodeDto;
+import org.graylog2.cluster.nodes.NodeService;
+import org.graylog2.cluster.preflight.DataNodeProvisioningConfig;
+import org.graylog2.cluster.preflight.DataNodeProvisioningService;
+
+import java.util.Map;
+
+@Api(value = "Certificate Provisioning for data node")
+@Path("/datanode/provision")
+@Produces(MediaType.APPLICATION_JSON)
+@RequiresAuthentication
+public class DataNodeProvisioningResource {
+
+    private final NodeService<DataNodeDto> nodeService;
+    private final DataNodeProvisioningService dataNodeProvisioningService;
+
+    @Inject
+    public DataNodeProvisioningResource(NodeService<DataNodeDto> nodeService, DataNodeProvisioningService dataNodeProvisioningService) {
+        this.nodeService = nodeService;
+        this.dataNodeProvisioningService = dataNodeProvisioningService;
+    }
+
+    @POST
+    @Path("/generate")
+    @NoAuditEvent("No Audit Event needed")
+    public void generate() {
+        final Map<String, DataNodeDto> activeDataNodes = nodeService.allActive();
+        activeDataNodes.values().forEach(node -> dataNodeProvisioningService.changeState(node.getNodeId(), DataNodeProvisioningConfig.State.CONFIGURED));
+    }
+
+
+}

graylog2-server/src/main/java/org/graylog2/rest/resources/RestResourcesModule.java

@@ -21,6 +21,7 @@
 import org.graylog2.Configuration;
 import org.graylog2.bootstrap.preflight.web.resources.CAResource;
 import org.graylog2.bootstrap.preflight.web.resources.CertificateRenewalResource;
+import org.graylog2.bootstrap.preflight.web.resources.DataNodeProvisioningResource;
 import org.graylog2.contentstream.rest.ContentStreamResource;
 import org.graylog2.plugin.inject.Graylog2Module;
 import org.graylog2.rest.resources.cluster.ClusterDeflectorResource;
@@ -162,6 +163,7 @@ protected void configure() {
         addSystemRestResource(DataNodeManagementResource.class);
         addSystemRestResource(RemoteReindexResource.class);
         addSystemRestResource(CAResource.class);
+        addSystemRestResource(DataNodeProvisioningResource.class);
     }
 
     private void addDebugResources() {

graylog2-server/src/test/java/org/graylog/plugins/views/storage/migration/state/machine/MigrationActionsAdapter.java

@@ -83,4 +83,13 @@ public boolean removalPolicyDoesNotExist() {
     public boolean caAndRemovalPolicyExist() {
         return false;
     }
+
+    @Override
+    public void provisionDataNodes() {
+    }
+
+    @Override
+    public boolean provisioningFinished() {
+        return false;
+    }
 }