PullRequest: 589 feat: 数据归档支持对象存储

Merge branch 'feat/dataArchiving_433_oss of [email protected]:oceanbase/oceanbase-developer-center.git into dev-4.3.3 https://code.alipay.com/oceanbase/oceanbase-developer-center/pull_requests/589 Signed-off-by: 晓康 <[email protected]> * feat: 数据归档支持对象存储 * feat: 删除调试代码、注释 * feat: 增加选择数据库逻辑 * feat: 数据归档源端数据库屏蔽对象存储、项目对象存储屏蔽操作列 * feat：删除log
oceanbase · Dec 9, 2024 · 9547ec8 · 9547ec8
1 parent 4aca0f1
commit 9547ec8
Show file tree

Hide file tree

Showing 53 changed files with 901 additions and 221 deletions.
diff --git a/src/common/datasource/fileSystem/index.tsx b/src/common/datasource/fileSystem/index.tsx
@@ -0,0 +1,48 @@
+import { ConnectType, TaskType } from '@/d.ts';
+import { IDataSourceModeConfig } from '../interface';
+import MySQLColumnExtra from '../oceanbase/MySQLColumnExtra';
+import { haveOCP } from '@/util/env';
+
+const CloudStorageConfig: IDataSourceModeConfig = {
+  connection: {
+    address: {
+      items: ['ip'],
+    },
+    account: false,
+    sys: false,
+    ssl: false,
+    disableURLParse: true,
+    cloudStorage: true,
+    disableExtraConfig: true,
+  },
+  features: {
+    task: [TaskType.DATA_ARCHIVE, TaskType.DATA_DELETE],
+    obclient: false,
+    recycleBin: false,
+    sessionManage: false,
+    sessionParams: false,
+    sqlExplain: false,
+    resourceTree: false,
+    export: {
+      fileLimit: false,
+      snapshot: false,
+    },
+  },
+};
+
+const ALIYUN: Record<ConnectType.OSS, IDataSourceModeConfig> = {
+  [ConnectType.OSS]: CloudStorageConfig,
+};
+const AWSS3: Record<ConnectType.S3A, IDataSourceModeConfig> = {
+  [ConnectType.S3A]: CloudStorageConfig,
+};
+
+const HUAWEI: Record<ConnectType.OBS, IDataSourceModeConfig> = {
+  [ConnectType.OBS]: CloudStorageConfig,
+};
+
+const QCLOUD: Record<ConnectType.COS, IDataSourceModeConfig> = {
+  [ConnectType.COS]: CloudStorageConfig,
+};
+
+export default { ALIYUN, AWSS3, HUAWEI, QCLOUD };
diff --git a/src/common/datasource/index.tsx b/src/common/datasource/index.tsx
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { ConnectType, ConnectionMode } from '@/d.ts';
+import { ConnectType, ConnectionMode, DatasourceGroup } from '@/d.ts';
 import { IDataSourceModeConfig } from './interface';
 import { IDataSourceType } from '@/d.ts/datasource';
 import obOracle from './oceanbase/oboracle';
@@ -23,6 +23,7 @@ import oracle from './oracle';
 import MySQL from './mysql';
 import Doris from './doris';
 import PG from './pg';
+import FileSystem from './fileSystem';
 import { ReactComponent as OBSvg } from '@/svgr/source_ob.svg';
 import { ReactComponent as DBOBSvg } from '@/svgr/database_oceanbase.svg';
 import { ReactComponent as MySQLSvg } from '@/svgr/mysql.svg';
@@ -34,8 +35,16 @@ import { ReactComponent as OracleSvg } from '@/svgr/oracle.svg';
 import { ReactComponent as DBOracleSvg } from '@/svgr/database_oracle.svg';
 import { DBType, BooleanOptionType } from '@/d.ts/database';
 import { ReactComponent as DBPGSvg } from '@/svgr/database_pg.svg';
+import { ReactComponent as OSSSvg } from '@/svgr/oss.svg';
+import { ReactComponent as DBOSSSvg } from '@/svgr/oss_file.svg';
+import { ReactComponent as COSSvg } from '@/svgr/COS.svg';
+import { ReactComponent as DBCOSSvg } from '@/svgr/cos_file.svg';
+import { ReactComponent as OBSSvg } from '@/svgr/OBS.svg';
+import { ReactComponent as DBOBSSvg } from '@/svgr/obs_file.svg';
+import { ReactComponent as S3Svg } from '@/svgr/S3.svg';
+import { ReactComponent as DBS3Svg } from '@/svgr/S3_file.svg';
 
-const _types: Map<
+export const _types: Map<
   IDataSourceType,
   {
     connectTypes: ConnectType[];
@@ -91,9 +100,57 @@ const _styles = {
       component: DBPGSvg,
     },
   },
+  [IDataSourceType.ALIYUNOSS]: {
+    icon: {
+      component: OSSSvg,
+      color: '#000000',
+    },
+    dbIcon: {
+      component: DBOSSSvg,
+    },
+  },
+  [IDataSourceType.HUAWEI]: {
+    icon: {
+      component: OBSSvg,
+      color: '#000000',
+    },
+    dbIcon: {
+      component: DBOBSSvg,
+    },
+  },
+  [IDataSourceType.AWSS3]: {
+    icon: {
+      component: S3Svg,
+      color: '#000000',
+    },
+    dbIcon: {
+      component: DBS3Svg,
+    },
+  },
+  [IDataSourceType.QCLOUD]: {
+    icon: {
+      component: COSSvg,
+      color: '#000000',
+    },
+    dbIcon: {
+      component: DBCOSSvg,
+    },
+  },
+};
+
+const _gruops = {
+  [IDataSourceType.OceanBase]: DatasourceGroup.OceanBaseDatabase,
+  [IDataSourceType.MySQL]: DatasourceGroup.OtherDatabase,
+  [IDataSourceType.Doris]: DatasourceGroup.OtherDatabase,
+  [IDataSourceType.Oracle]: DatasourceGroup.OtherDatabase,
+  [IDataSourceType.PG]: DatasourceGroup.OtherDatabase,
+  [IDataSourceType.ALIYUNOSS]: DatasourceGroup.FileSystem,
+  [IDataSourceType.AWSS3]: DatasourceGroup.FileSystem,
+  [IDataSourceType.HUAWEI]: DatasourceGroup.FileSystem,
+  [IDataSourceType.QCLOUD]: DatasourceGroup.FileSystem,
 };
 
-const connectType2Ds: Map<ConnectType, IDataSourceType> = new Map();
+export const connectType2Ds: Map<ConnectType, IDataSourceType> = new Map();
 
 function register(
   dataSourceType: IDataSourceType,
@@ -135,6 +192,10 @@ register(IDataSourceType.MySQL, MySQL);
 register(IDataSourceType.Doris, Doris);
 register(IDataSourceType.Oracle, oracle);
 register(IDataSourceType.PG, PG);
+register(IDataSourceType.ALIYUNOSS, FileSystem.ALIYUN);
+register(IDataSourceType.AWSS3, FileSystem.AWSS3);
+register(IDataSourceType.HUAWEI, FileSystem.HUAWEI);
+register(IDataSourceType.QCLOUD, FileSystem.QCLOUD);
 
 function getAllConnectTypes(ds?: IDataSourceType): ConnectType[] {
   if (!ds) {
@@ -188,6 +249,14 @@ function getDataSourceStyleByConnectType(ct: ConnectType) {
   return getDataSourceStyle(connectType2Ds[ct]);
 }
 
+function getDataSourceGroup(ds: IDataSourceType) {
+  return _gruops[ds];
+}
+
+function getDataSourceGroupByConnectType(ct: ConnectType) {
+  return getDataSourceGroup(connectType2Ds[ct]);
+}
+
 function getDsByConnectType(ct: ConnectType) {
   return connectType2Ds[ct];
 }
@@ -202,6 +271,7 @@ export {
   getDataSourceModeConfigByConnectionMode,
   getDataSourceStyle,
   getDataSourceStyleByConnectType,
+  getDataSourceGroupByConnectType,
   getDefaultConnectType,
   getDsByConnectType,
   getAllDBTypes,

diff --git a/src/common/datasource/interface.ts b/src/common/datasource/interface.ts
@@ -116,7 +116,7 @@ interface IProcedureConfig {
 export interface IDataSourceModeConfig {
   priority?: number;
   connection: {
-    address: {
+    address?: {
       items: ('ip' | 'port' | 'cluster' | 'tenant' | 'sid' | 'catalogName')[];
     };
     account: boolean;
@@ -127,6 +127,8 @@ export interface IDataSourceModeConfig {
     jdbcDoc?: string;
     disableURLParse?: boolean;
     unionUser?: boolean;
+    cloudStorage?: boolean;
+    disableExtraConfig?: boolean;
   };
   features: {
     task: TaskType[];
@@ -148,13 +150,13 @@ export interface IDataSourceModeConfig {
       snapshot: boolean;
     };
   };
-  schema: {
+  schema?: {
     table: ICreateTableConfig;
     func: IFunctionConfig;
     proc: IProcedureConfig;
     innerSchema: string[];
   };
-  sql: {
+  sql?: {
     language: string;
     escapeChar: string;
     plParamMode?: 'text' | 'list';

diff --git a/src/common/network/connection.ts b/src/common/network/connection.ts
@@ -97,7 +97,6 @@ function generateConnectionParams(formData: Partial<IConnectionFormData>, isHide
  */
 export async function createConnection(formData: Partial<IDatasource>, isHiden?: boolean) {
   const params: Partial<IConnection> = generateConnectionParams(formData, isHiden);
-
   const requestParams = {
     wantCatchError: false,
     holdErrorTip: true,
@@ -326,7 +325,7 @@ export async function getSessionStatus(sessionId?: string): Promise<{
     obVersion: string;
   };
   session: ISessionStatus;
-  killCurrentQuerySupported: boolean
+  killCurrentQuerySupported: boolean;
 }> {
   const sid = generateSessionSid(sessionId);
   const res = await request.get(`/api/v2/datasource/sessions/${sessionId}/status`);

diff --git a/src/common/network/sql/executeSQL.tsx b/src/common/network/sql/executeSQL.tsx
@@ -19,8 +19,12 @@ import type { ISqlExecuteResult, IExecutingInfo } from '@/d.ts';
 import { EStatus, ISqlExecuteResultStatus } from '@/d.ts';
 import request from '@/util/request';
 import { generateDatabaseSid, generateSessionSid } from '../pathUtil';
-import { executeSQLPreHandle, IExecuteSQLParams, IExecuteTaskResult, ISQLExecuteTask } from './preHandle';
-
+import {
+  executeSQLPreHandle,
+  IExecuteSQLParams,
+  IExecuteTaskResult,
+  ISQLExecuteTask,
+} from './preHandle';
 
 class Task {
   public result: ISqlExecuteResult[] = [];
@@ -176,7 +180,6 @@ export default async function executeSQL(
   });
   const taskInfo: ISQLExecuteTask = res?.data;
 
-
   const {
     pass,
     data: preHandleData,

diff --git a/src/common/network/sql/preHandle.tsx b/src/common/network/sql/preHandle.tsx
@@ -38,7 +38,7 @@ export interface ISQLExecuteTask {
   violatedRules: IRule[];
   unauthorizedDBResources: IUnauthorizedDBResources[];
   errorMessage?: string;
-  approvalRequired?: boolean
+  approvalRequired?: boolean;
 }
 
 /**
@@ -55,15 +55,15 @@ export interface IExecuteTaskResult {
   unauthorizedDBResources?: IUnauthorizedDBResources[];
   unauthorizedSql?: string;
   errorMessage?: string;
-  approvalRequired?: boolean
+  approvalRequired?: boolean;
 }
 
 export function executeSQLPreHandle(
   taskInfo: ISQLExecuteTask,
   params: IExecuteSQLParams | IExecutePLForMysqlParams | string,
   needModal: boolean,
   sessionId: string,
-  handleUnauthInModal?: boolean
+  handleUnauthInModal?: boolean,
 ): {
   data: any;
   lintResultSet: ISQLLintReuslt[];
@@ -146,7 +146,7 @@ export function executeSQLPreHandle(
         sessionId,
         lintResultSet,
         unauthorizedDBResources,
-        status: unauthorizedDBResources?.length ? EStatus.DISABLED :lintStatus, 
+        status: unauthorizedDBResources?.length ? EStatus.DISABLED : lintStatus,
         onSave: () => {
           // 关闭SQL确认窗口打开新建数据库变更抽屉
           modal.updateWorkSpaceExecuteSQLModalProps();
@@ -167,7 +167,6 @@ export function executeSQLPreHandle(
   const requestId = taskInfo?.requestId;
   const sqls = taskInfo?.sqls;
   if (!requestId || !sqls?.length) {
-
     return {
       data: null,
       lintResultSet,

diff --git a/src/common/network/table/helper.ts b/src/common/network/table/helper.ts
@@ -535,11 +535,12 @@ export function convertServerTablePartitionToTablePartition(
     }
   };
 
-  const handleColumns = (partition)=> partition?.partitionOption?.columnNames?.map((item) => {
-    return {
-      columnName: item,
-    };
-  })
+  const handleColumns = (partition) =>
+    partition?.partitionOption?.columnNames?.map((item) => {
+      return {
+        columnName: item,
+      };
+    });
 
   switch (partType) {
     case IPartitionType.HASH: {
@@ -549,7 +550,7 @@ export function convertServerTablePartitionToTablePartition(
         expression: partition?.partitionOption?.expression,
         columns: handleColumns(partition),
         partitions: handlePartitions(partType, dbMode, partition),
-        subpartitionTemplated: subpartitionTemplated
+        subpartitionTemplated: subpartitionTemplated,
       };
       break;
     }
@@ -560,7 +561,7 @@ export function convertServerTablePartitionToTablePartition(
         columns: handleColumns(partition),
         expression: partition?.partitionOption?.expression,
         partitions: handlePartitions(partType, dbMode, partition),
-        subpartitionTemplated: subpartitionTemplated
+        subpartitionTemplated: subpartitionTemplated,
       };
       break;
     }
@@ -570,7 +571,7 @@ export function convertServerTablePartitionToTablePartition(
         expression: partition?.partitionOption?.expression,
         columns: handleColumns(partition),
         partitions: handlePartitions(partType, dbMode, partition),
-        subpartitionTemplated: subpartitionTemplated
+        subpartitionTemplated: subpartitionTemplated,
       };
       break;
     }
@@ -580,7 +581,7 @@ export function convertServerTablePartitionToTablePartition(
         columns: handleColumns(partition),
         expression: partition?.partitionOption?.expression,
         partitions: handlePartitions(partType, dbMode, partition),
-        subpartitionTemplated: subpartitionTemplated
+        subpartitionTemplated: subpartitionTemplated,
       };
       break;
     }
@@ -590,7 +591,7 @@ export function convertServerTablePartitionToTablePartition(
         expression: partition?.partitionOption?.expression,
         columns: handleColumns(partition),
         partitions: handlePartitions(partType, dbMode, partition),
-        subpartitionTemplated: subpartitionTemplated
+        subpartitionTemplated: subpartitionTemplated,
       };
       break;
     }
@@ -600,7 +601,7 @@ export function convertServerTablePartitionToTablePartition(
         columns: handleColumns(partition),
         expression: partition?.partitionOption?.expression,
         partitions: handlePartitions(partType, dbMode, partition),
-        subpartitionTemplated: subpartitionTemplated
+        subpartitionTemplated: subpartitionTemplated,
       };
       break;
     }