Add data node migration wizard

changelog/unreleased/pr-18079.toml

@@ -0,0 +1,6 @@
+type = "added"
+message = "add datanode migration ui"
+
+issues = [""]
+pulls = ["18079"]
+

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java

@@ -161,13 +161,15 @@ public void provisionAndStartDataNodes() {
 
     @Override
     public boolean provisioningFinished() {
-        return nodeService.allActive().values().stream().allMatch(node -> node.getDataNodeStatus() == DataNodeStatus.PREPARED);
+        return nodeService.allActive().values().stream().allMatch(node -> dataNodeProvisioningService.getPreflightConfigFor(node.getNodeId())
+                .map(dn -> dn.state() == DataNodeProvisioningConfig.State.STARTUP_PREPARED)
+                .orElse(false));
     }
 
     @Override
     public void startDataNodes() {
         final Map<String, DataNodeDto> activeDataNodes = nodeService.allActive();
-        activeDataNodes.values().forEach(node -> dataNodeProvisioningService.changeState(node.getNodeId(), DataNodeProvisioningConfig.State.STARTUP_REQUESTED));
+        activeDataNodes.values().forEach(node -> dataNodeProvisioningService.changeState(node.getNodeId(), DataNodeProvisioningConfig.State.STARTUP_TRIGGER));
     }
 
     @Override

graylog2-web-interface/src/components/datanode/Constants.ts

@@ -0,0 +1,200 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+
+export const MIGRATION_STEP = {
+  CA_CONFIGURATION: {
+    key: 'CA_CONFIGURATION',
+    description: 'Configure a certificate authority',
+  },
+  RENEWAL_POLICY_CONFIGURATION: {
+    key: 'RENEWAL_POLICY_CONFIGURATION',
+    description: 'Configure a renewal policy',
+  },
+  COMPATIBILITY_CHECK: {
+    key: 'COMPATIBILITY_CHECK',
+    description: 'Check OpenSearch compatibility with datanode',
+  },
+  MANUAL_MIGRATION_STEP: {
+    key: 'MANUAL_MIGRATION_STEP',
+    description: 'Migration steps.',
+  },
+  MIGRATION_FINISHED: {
+    key: 'MIGRATION_FINISHED',
+    description: 'Migration finished',
+  },
+} as const;
+
+export const MIGRATION_STATE = {
+  NEW: {
+    key: 'NEW',
+    description: 'Migration',
+  },
+  MIGRATION_WELCOME_PAGE: {
+    key: 'MIGRATION_WELCOME_PAGE',
+    description: 'Welcome',
+  },
+  CA_CREATION_PAGE: {
+    key: 'CA_CREATION_PAGE',
+    description: 'Certificate authority',
+  },
+  RENEWAL_POLICY_CREATION_PAGE: {
+    key: 'RENEWAL_POLICY_CREATION_PAGE',
+    description: 'Certificate renewal policy',
+  },
+  MIGRATION_SELECTION_PAGE: {
+    key: 'MIGRATION_SELECTION_PAGE',
+    description: 'Migration steps',
+  },
+  ROLLING_UPGRADE_MIGRATION_WELCOME_PAGE: {
+    key: 'ROLLING_UPGRADE_MIGRATION_WELCOME_PAGE',
+    description: 'Welcome to Rolling upgrade migration',
+  },
+  ASK_TO_SHUTDOWN_OLD_CLUSTER: {
+    key: 'ASK_TO_SHUTDOWN_OLD_CLUSTER',
+    description: 'Shut down old cluster',
+  },
+  MANUALLY_REMOVE_OLD_CONNECTION_STRING_FROM_CONFIG: {
+    key: 'MANUALLY_REMOVE_OLD_CONNECTION_STRING_FROM_CONFIG',
+    description: 'Remove connection string',
+  },
+  MESSAGE_PROCESSING_STOP: {
+    key: 'MESSAGE_PROCESSING_STOP',
+    description: 'Stop message processing',
+  },
+  REPLACE_CLUSTER: {
+    key: 'REPLACE_CLUSTER',
+    description: 'Replace existing cluster',
+  },
+  RESTART_GRAYLOG: {
+    key: 'RESTART_GRAYLOG',
+    description: 'Update configuration file and restart Graylog',
+  },
+  REMOTE_REINDEX_WELCOME_PAGE: {
+    key: 'REMOTE_REINDEX_WELCOME_PAGE',
+    description: 'Remote reindexing migration',
+  },
+  PROVISION_DATANODE_CERTIFICATES_PAGE: {
+    key: 'PROVISION_DATANODE_CERTIFICATES_PAGE',
+    description: 'Provision Data Node with certificates',
+  },
+  PROVISION_DATANODE_CERTIFICATES_RUNNING: {
+    key: 'PROVISION_DATANODE_CERTIFICATES_RUNNING',
+    description: 'Provision Data Node with certificates running',
+  },
+  EXISTING_DATA_MIGRATION_QUESTION_PAGE: {
+    key: 'EXISTING_DATA_MIGRATION_QUESTION_PAGE',
+    description: 'Migrate existing data question',
+  },
+  MIGRATE_EXISTING_DATA: {
+    key: 'MIGRATE_EXISTING_DATA',
+    description: 'Migrate existing data',
+  },
+  REMOTE_REINDEX_RUNNING: {
+    key: 'REMOTE_REINDEX_RUNNING',
+    description: 'Remote reindexing migration running',
+  },
+  DIRECTORY_COMPATIBILITY_CHECK_PAGE: {
+    key: 'DIRECTORY_COMPATIBILITY_CHECK_PAGE',
+    description: 'Directory compatibility check',
+  },
+  DIRECTORY_COMPATIBILITY_CHECK_PAGE2: {
+    key: 'DIRECTORY_COMPATIBILITY_CHECK_PAGE2',
+    description: 'Directory compatibility check',
+  },
+  PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES: {
+    key: 'PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES',
+    description: 'Certificate provisioning overview',
+  },
+  PROVISION_ROLLING_UPGRADE_NODES_RUNNING: {
+    key: 'PROVISION_ROLLING_UPGRADE_NODES_RUNNING',
+    description: 'Provisionning Data Nodes certificate.',
+  },
+  JOURNAL_SIZE_DOWNTIME_WARNING: {
+    key: 'JOURNAL_SIZE_DOWNTIME_WARNING',
+    description: 'Journal size downtime warning',
+  },
+  FAILED: {
+    key: 'FAILED',
+    description: 'Migration failed',
+  },
+  FINISHED: {
+    key: 'FINISHED',
+    description: 'Migration finished',
+  },
+} as const;
+
+export const ROLLING_UPGRADE_MIGRATION_STEPS = [
+  MIGRATION_STATE.ROLLING_UPGRADE_MIGRATION_WELCOME_PAGE.key,
+  MIGRATION_STATE.DIRECTORY_COMPATIBILITY_CHECK_PAGE2.key,
+  MIGRATION_STATE.PROVISION_ROLLING_UPGRADE_NODES_WITH_CERTIFICATES.key,
+  MIGRATION_STATE.PROVISION_ROLLING_UPGRADE_NODES_RUNNING.key,
+  MIGRATION_STATE.JOURNAL_SIZE_DOWNTIME_WARNING.key,
+  MIGRATION_STATE.MESSAGE_PROCESSING_STOP.key,
+  MIGRATION_STATE.RESTART_GRAYLOG.key,
+];
+export const REMOTE_REINDEXING_MIGRATION_STEPS = [
+  MIGRATION_STATE.REMOTE_REINDEX_WELCOME_PAGE.key,
+  MIGRATION_STATE.PROVISION_DATANODE_CERTIFICATES_PAGE.key,
+  MIGRATION_STATE.PROVISION_DATANODE_CERTIFICATES_RUNNING.key,
+  MIGRATION_STATE.EXISTING_DATA_MIGRATION_QUESTION_PAGE.key,
+  MIGRATION_STATE.MIGRATE_EXISTING_DATA.key,
+  MIGRATION_STATE.REMOTE_REINDEX_RUNNING.key,
+  MIGRATION_STATE.ASK_TO_SHUTDOWN_OLD_CLUSTER.key,
+  MIGRATION_STATE.MANUALLY_REMOVE_OLD_CONNECTION_STRING_FROM_CONFIG.key,
+];
+
+export const MIGRATION_WIZARD_STEPS = [
+  MIGRATION_STATE.NEW.key,
+  MIGRATION_STATE.MIGRATION_WELCOME_PAGE.key,
+  MIGRATION_STATE.DIRECTORY_COMPATIBILITY_CHECK_PAGE.key,
+  MIGRATION_STATE.CA_CREATION_PAGE.key,
+  MIGRATION_STATE.RENEWAL_POLICY_CREATION_PAGE.key,
+  MIGRATION_STATE.MIGRATION_SELECTION_PAGE.key,
+  MIGRATION_STATE.FINISHED.key,
+];
+
+export const MIGRATION_ACTIONS = {
+  SHOW_DIRECTORY_COMPATIBILITY_CHECK: {
+    key: 'SHOW_DIRECTORY_COMPATIBILITY_CHECK',
+    label: 'Directory compatibility check',
+  },
+  SKIP_DIRECTORY_COMPATIBILITY_CHECK: {
+    key: 'SKIP_DIRECTORY_COMPATIBILITY_CHECK',
+    label: 'Skip directory compatibility check',
+  },
+  SHOW_RENEWAL_POLICY_CREATION: {
+    key: 'SHOW_RENEWAL_POLICY_CREATION',
+    label: 'Configure certificat renewal policy',
+  },
+  SHOW_MIGRATION_SELECTION: {
+    key: 'SHOW_MIGRATION_SELECTION',
+    label: 'Go to migration steps',
+  },
+  PROVISION_DATANODE_CERTIFICATES: {
+    key: 'PROVISION_DATANODE_CERTIFICATES',
+    label: 'Provision Data Nodes with certificates',
+  },
+  SKIP_EXISTING_DATA_MIGRATION: {
+    key: 'SKIP_EXISTING_DATA_MIGRATION',
+    label: 'Skip existing data migration',
+  },
+  RETRY_MIGRATE_EXISTING_DATA: {
+    key: 'RETRY_MIGRATE_EXISTING_DATA',
+    label: 'Retry migrate existing data',
+  },
+};
+export default MIGRATION_STEP;

graylog2-web-interface/src/components/datanode/DataNodeConfiguration/CAConfiguration.tsx

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+import React from 'react';
+
+import { Tabs, Tab } from 'components/bootstrap';
+import CACreateForm from 'components/datanode/DataNodeConfiguration/CACreateForm';
+import CAUpload from 'components/datanode/DataNodeConfiguration/CAUpload';
+
+const TAB_KEYS = ['create', 'upload'];
+const CAConfiguration = () => (
+  <>
+    <h2>Configure Certificate Authority</h2>
+    <p>
+      In this first step you can either upload or create a new certificate authority.<br />
+      Using it we can provision your Data Nodes with certificates easily.
+    </p>
+    <Tabs defaultActiveKey={TAB_KEYS[0]} id="ca-configurations">
+      <Tab eventKey={TAB_KEYS[0]} title="Create new CA">
+        <CACreateForm />
+      </Tab>
+      <Tab eventKey={TAB_KEYS[1]} title="Upload CA">
+        <CAUpload />
+      </Tab>
+    </Tabs>
+  </>
+);
+export default CAConfiguration;

graylog2-web-interface/src/components/datanode/DataNodeConfiguration/CACreateForm.test.tsx

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+import React from 'react';
+import { render, screen, waitFor } from 'wrappedTestingLibrary';
+import userEvent from '@testing-library/user-event';
+import DefaultQueryClientProvider from 'DefaultQueryClientProvider';
+
+import { asMock, StoreMock as MockStore } from 'helpers/mocking';
+import fetch from 'logic/rest/FetchProvider';
+import UserNotification from 'util/UserNotification';
+
+import CACreateForm from './CACreateForm';
+
+jest.mock('logic/rest/FetchProvider', () => jest.fn(() => Promise.resolve()));
+jest.mock('stores/sessions/SessionStore', () => ({ SessionStore: MockStore(['isLoggedIn', jest.fn()]) }));
+jest.mock('stores/system/SystemStore', () => ({ SystemStore: MockStore() }));
+
+jest.mock('util/UserNotification', () => ({
+  error: jest.fn(),
+  success: jest.fn(),
+}));
+
+const logger = {
+  // eslint-disable-next-line no-console
+  log: console.log,
+  // eslint-disable-next-line no-console
+  warn: console.warn,
+  error: () => {},
+};
+
+describe('CACreateForm', () => {
+  beforeEach(() => {
+    asMock(fetch).mockReturnValue(Promise.resolve());
+  });
+
+  const submitForm = async () => {
+    userEvent.click(await screen.findByRole('button', { name: /Create CA/i }));
+  };
+
+  it('should create CA', async () => {
+    render(<CACreateForm />);
+
+    await submitForm();
+
+    await waitFor(() => expect(fetch).toHaveBeenCalledWith(
+      'POST',
+      expect.stringContaining('/ca/create'),
+      { organization: 'Graylog CA' },
+      false,
+    ));
+
+    expect(UserNotification.success).toHaveBeenCalledWith('CA created successfully');
+  });
+
+  it('should show error when CA creation fails', async () => {
+    asMock(fetch).mockImplementation(() => Promise.reject(new Error('Error')));
+
+    render((
+      <DefaultQueryClientProvider options={{ logger }}>
+        <CACreateForm />
+      </DefaultQueryClientProvider>
+    ));
+
+    await submitForm();
+
+    await waitFor(() => expect(fetch).toHaveBeenCalledWith(
+      'POST',
+      expect.stringContaining('/ca/create'),
+      { organization: 'Graylog CA' },
+      false,
+    ));
+
+    expect(UserNotification.error).toHaveBeenCalledWith('CA creation failed with error: Error: Error');
+  });
+});