feat: support recommended backup strategies and row backups

sqle/api/app.go

@@ -232,7 +232,7 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
 		v1ProjectOpRouter.PATCH("/:project_name/workflows/:workflow_name/", DeprecatedBy(apiV2))
 		v1ProjectOpRouter.POST("/:project_name/workflows/:workflow_id/tasks/:task_id/order_file", v1.UpdateSqlFileOrderByWorkflowV1)
 		v1ProjectOpRouter.GET("/:project_name/workflows/:workflow_id/backup_sqls", v1.GetBackupSqlList)
-		v1ProjectOpRouter.POST("/:project_name/workflows/:workflow_id/rollback_workflows", v1.CreateRollbackWorkflow)
+		v1ProjectOpRouter.POST("/:project_name/workflows/:workflow_id/create_rollback_workflow", v1.CreateRollbackWorkflow)
 
 		// sql version
 		v1ProjectOpRouter.POST("/:project_name/sql_versions/:sql_version_id/batch_release_workflows", v1.BatchReleaseWorkflows)

sqle/api/controller/v1/backup.go

@@ -102,37 +102,3 @@ type BackupSqlListRes struct {
 func GetBackupSqlList(c echo.Context) error {
 	return getBackupSqlList(c)
 }
-
-type CreateRollbackWorkflowReq struct {
-	Subject        string `json:"workflow_subject" form:"workflow_subject" valid:"required,name"`
-	Desc           string `json:"desc" form:"desc"`
-	SqlVersionID   *uint  `json:"sql_version_id" form:"sql_version_id"`
-	TaskIds        []uint `json:"task_ids" form:"task_ids" valid:"required"`
-	RollbackSqlIds []uint `json:"rollback_sql_ids" form:"rollback_sql_ids" valid:"required"`
-}
-
-type CreateRollbackWorkflowRes struct {
-	controller.BaseRes
-	Data *CreateRollbackWorkflowResData `json:"data"`
-}
-
-type CreateRollbackWorkflowResData struct {
-	WorkflowID string `json:"workflow_id"`
-}
-
-// CreateRollbackWorkflow
-// @Summary 创建回滚工单
-// @Description create rollback workflow
-// @Accept json
-// @Produce json
-// @Tags workflow
-// @Id CreateRollbackWorkflow
-// @Security ApiKeyAuth
-// @Param instance body v1.CreateRollbackWorkflowReq true "create rollback workflow request"
-// @Param project_name path string true "project name"
-// @Param workflow_id path string true "origin workflow id to rollback"
-// @Success 200 {object} CreateRollbackWorkflowRes
-// @router /v1/projects/{project_name}/workflows/{workflow_id}/rollback_workflows [post]
-func CreateRollbackWorkflow(c echo.Context) error {
-	return createRollbackWorkflow(c)
-}

sqle/api/controller/v1/task.go

@@ -64,7 +64,7 @@ type AuditTaskResV1 struct {
 	FileOrderMethod            string          `json:"file_order_method,omitempty"`
 	ExecMode                   string          `json:"exec_mode,omitempty"`
 	EnableBackup               bool            `json:"enable_backup"`
-	BackupConflictWithInstance bool            `json:"backup_conflict_with_instance"`
+	BackupConflictWithInstance bool            `json:"backup_conflict_with_instance"` // 当数据源备份开启，工单备份关闭，则需要提示审核人工单备份策略与数据源备份策略不一致
 	AuditFiles                 []AuditFileResp `json:"audit_files,omitempty"`
 }
 

sqle/api/controller/v1/workflow.go

@@ -1669,3 +1669,37 @@ func GetWorkflowTaskAuditFile(c echo.Context) error {
 	}
 	return c.NoContent(http.StatusOK)
 }
+
+type CreateRollbackWorkflowReq struct {
+	Subject        string `json:"workflow_subject" form:"workflow_subject" valid:"required,name"`
+	Desc           string `json:"desc" form:"desc"`
+	SqlVersionID   *uint  `json:"sql_version_id" form:"sql_version_id"`
+	TaskIds        []uint `json:"task_ids" form:"task_ids" valid:"required"`
+	RollbackSqlIds []uint `json:"rollback_sql_ids" form:"rollback_sql_ids" valid:"required"`
+}
+
+type CreateRollbackWorkflowRes struct {
+	controller.BaseRes
+	Data *CreateRollbackWorkflowResData `json:"data"`
+}
+
+type CreateRollbackWorkflowResData struct {
+	WorkflowID string `json:"workflow_id"`
+}
+
+// CreateRollbackWorkflow
+// @Summary 创建回滚工单
+// @Description create rollback workflow
+// @Accept json
+// @Produce json
+// @Tags workflow
+// @Id CreateRollbackWorkflow
+// @Security ApiKeyAuth
+// @Param instance body v1.CreateRollbackWorkflowReq true "create rollback workflow request"
+// @Param project_name path string true "project name"
+// @Param workflow_id path string true "origin workflow id to rollback"
+// @Success 200 {object} CreateRollbackWorkflowRes
+// @router /v1/projects/{project_name}/workflows/{workflow_id}/create_rollback_workflow [post]
+func CreateRollbackWorkflow(c echo.Context) error {
+	return createRollbackWorkflow(c)
+}

sqle/docs/docs.go

@@ -7197,7 +7197,7 @@ var doc = `{
                 }
             }
         },
-        "/v1/projects/{project_name}/workflows/{workflow_id}/rollback_workflows": {
+        "/v1/projects/{project_name}/workflows/{workflow_id}/create_rollback_workflow": {
             "post": {
                 "security": [
                     {
@@ -11639,6 +11639,7 @@ var doc = `{
                     ]
                 },
                 "backup_conflict_with_instance": {
+                    "description": "当数据源备份开启，工单备份关闭，则需要提示审核人工单备份策略与数据源备份策略不一致",
                     "type": "boolean"
                 },
                 "enable_backup": {

sqle/docs/swagger.json

@@ -7181,7 +7181,7 @@
                 }
             }
         },
-        "/v1/projects/{project_name}/workflows/{workflow_id}/rollback_workflows": {
+        "/v1/projects/{project_name}/workflows/{workflow_id}/create_rollback_workflow": {
             "post": {
                 "security": [
                     {
@@ -11623,6 +11623,7 @@
                     ]
                 },
                 "backup_conflict_with_instance": {
+                    "description": "当数据源备份开启，工单备份关闭，则需要提示审核人工单备份策略与数据源备份策略不一致",
                     "type": "boolean"
                 },
                 "enable_backup": {

sqle/docs/swagger.yaml

@@ -389,6 +389,7 @@ definitions:
         - ""
         type: string
       backup_conflict_with_instance:
+        description: 当数据源备份开启，工单备份关闭，则需要提示审核人工单备份策略与数据源备份策略不一致
         type: boolean
       enable_backup:
         type: boolean
@@ -10524,7 +10525,7 @@ paths:
       summary: 获取工单下所有回滚SQL的列表
       tags:
       - workflow
-  /v1/projects/{project_name}/workflows/{workflow_id}/rollback_workflows:
+  /v1/projects/{project_name}/workflows/{workflow_id}/create_rollback_workflow:
     post:
       consumes:
       - application/json

sqle/driver/mysql/backup_ce.go

@@ -0,0 +1,21 @@
+//go:build !enterprise
+// +build !enterprise
+
+package mysql
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/actiontech/sqle/sqle/driver"
+)
+
+var ErrUnsupportedBackup error = fmt.Errorf("backup is unsupported for sqle community version")
+
+func (i *MysqlDriverImpl) Backup(ctx context.Context, backupStrategy string, sql string) (BackupSql []string, ExecuteInfo string, err error) {
+	return nil, "", ErrUnsupportedBackup
+}
+
+func (i *MysqlDriverImpl) RecommendBackupStrategy(ctx context.Context, sql string) (*driver.RecommendBackupStrategyRes, error) {
+	return nil, ErrUnsupportedBackup
+}

sqle/driver/mysql/mysql.go

@@ -99,15 +99,10 @@ func (inspect *MysqlDriverImpl) applyConfig(cfg *driverV2.Config) {
 	inspect.isOfflineAudit = cfg.DSN == nil
 
 	inspect.cnf = &Config{
-		DMLRollbackMaxRows: 1000,// TODO 暂时将备份影响行数上限设置为1000，后续需要将对应该配置项的规则移除。备份影响行数上限设置将会在备份任务中设置。
 		DDLOSCMinSize:      -1,
 		DDLGhostMinSize:    -1,
 	}
 	for _, rule := range cfg.Rules {
-		if rule.Name == rulepkg.ConfigDMLRollbackMaxRows {
-			max := rule.Params.GetParam(rulepkg.DefaultSingleParamKeyName).Int()
-			inspect.cnf.DMLRollbackMaxRows = int64(max)
-		}
 		if rule.Name == rulepkg.ConfigDDLOSCMinSize {
 			min := rule.Params.GetParam(rulepkg.DefaultSingleParamKeyName).Int()
 			inspect.cnf.DDLOSCMinSize = int64(min)
@@ -631,7 +626,7 @@ func (p *PluginProcessor) GetDriverMetas() (*driverV2.DriverMetas, error) {
 	for i := range rulepkg.RuleHandlers {
 		allRules[i] = &rulepkg.RuleHandlers[i].Rule
 	}
-	return &driverV2.DriverMetas{
+	metas := &driverV2.DriverMetas{
 		PluginName:               driverV2.DriverTypeMySQL,
 		DatabaseDefaultPort:      3306,
 		Logo:                     logo,
@@ -648,7 +643,9 @@ func (p *PluginProcessor) GetDriverMetas() (*driverV2.DriverMetas, error) {
 			driverV2.OptionalExecBatch,
 			driverV2.OptionalModuleI18n,
 		},
-	}, nil
+	}
+	addOptionModules(metas)
+	return metas, nil
 }
 
 func (p *PluginProcessor) Open(l *logrus.Entry, cfg *driverV2.Config) (driver.Plugin, error) {

sqle/driver/mysql/mysql_ce.go

@@ -21,3 +21,5 @@ func (i *MysqlDriverImpl) GetDatabaseDiffModifySQL(ctx context.Context, calibrat
 func (i *MysqlDriverImpl) GetDatabaseObjectDDL(ctx context.Context, objInfos []*driverV2.DatabasSchemaInfo) ([]*driverV2.DatabaseSchemaObjectResult, error) {
 	return nil, fmt.Errorf("only support Query in enterprise edition")
 }
+
+func addOptionModules(metas *driverV2.DriverMetas) {}

sqle/driver/mysql/rule/rule.go

@@ -206,7 +206,6 @@ const (
 
 // inspector config code
 const (
-	ConfigDMLRollbackMaxRows       = "dml_rollback_max_rows"
 	ConfigDDLOSCMinSize            = "ddl_osc_min_size"
 	ConfigDDLGhostMinSize          = "ddl_ghost_min_size"
 	ConfigOptimizeIndexEnabled     = "optimize_index_enabled"

sqle/driver/mysql/rule/rule_list.go

@@ -23,25 +23,6 @@ var sourceRuleHandlers = []*SourceHandler{
 		AllowOffline: false,
 		Func:         notAllowInsertAutoincrement,
 	},
-	{
-		Rule: SourceRule{
-			Name:       ConfigDMLRollbackMaxRows,
-			Desc:       plocale.ConfigDMLRollbackMaxRowsDesc,
-			Annotation: plocale.ConfigDMLRollbackMaxRowsAnnotation,
-			//Value:    "1000",
-			Level:    driverV2.RuleLevelNotice,
-			Category: plocale.RuleTypeGlobalConfig,
-			Params: []*SourceParam{
-				{
-					Key:   DefaultSingleParamKeyName,
-					Value: "1000",
-					Desc:  plocale.ConfigDMLRollbackMaxRowsParams1,
-					Type:  params.ParamTypeInt,
-				},
-			},
-		},
-		Func: nil,
-	},
 	{
 		Rule: SourceRule{
 			Name:       ConfigDDLOSCMinSize,

sqle/driver/plugin_adapter_v1.go

@@ -129,6 +129,14 @@ type PluginImplV1 struct {
 	driverV1.DriverManager
 }
 
+func (d *PluginImplV1) Backup(ctx context.Context, backupStrategy string, sql string) (backupSql []string, executeInfo string, err error) {
+	return nil, "", nil
+}
+
+func (d *PluginImplV1) RecommendBackupStrategy(ctx context.Context, sql string) (*RecommendBackupStrategyRes, error) {
+	return nil, nil
+}
+
 func (p *PluginImplV1) Close(ctx context.Context) {
 	p.DriverManager.Close(ctx)
 }

sqle/driver/plugin_adapter_v2.go

@@ -174,6 +174,30 @@ type PluginImplV2 struct {
 	meta    *driverV2.DriverMetas
 }
 
+func (s *PluginImplV2) Backup(ctx context.Context, backupStrategy string, sql string) (BackupSql []string, ExecuteInfo string, err error) {
+	api := "Backup"
+	s.preLog(api)
+	var strategy protoV2.BackupStrategy = protoV2.BackupStrategy_ReverseSql
+	switch backupStrategy {
+	case "reverse_sql":
+		strategy = protoV2.BackupStrategy_ReverseSql
+	}
+	resp, err := s.client.Backup(ctx, &protoV2.BackupReq{
+		Session:        s.Session,
+		BackupStrategy: strategy,
+		Sql:            sql,
+	})
+	s.afterLog(api, err)
+	if err != nil {
+		return nil, "", err
+	}
+	return resp.GetBackupSql(), resp.GetExecuteInfo(), nil
+}
+
+func (p *PluginImplV2) RecommendBackupStrategy(ctx context.Context, sql string) (*RecommendBackupStrategyRes, error) {
+	return nil, nil
+}
+
 func (s *PluginImplV2) preLog(ApiName string) {
 	s.l.Infof("starting call plugin interface [%s]", ApiName)
 }

sqle/driver/plugin_interface.go

@@ -46,6 +46,17 @@ type Plugin interface {
 	GetDatabaseObjectDDL(ctx context.Context, objInfos []*driverV2.DatabasSchemaInfo) ([]*driverV2.DatabaseSchemaObjectResult, error)
 
 	GetDatabaseDiffModifySQL(ctx context.Context, calibratedDSN *driverV2.DSN, objInfos []*driverV2.DatabasCompareSchemaInfo) ([]*driverV2.DatabaseDiffModifySQLResult, error)
+
+	Backup(ctx context.Context, backupStrategy string, sql string) (BackupSql []string, ExecuteInfo string, err error)
+
+	RecommendBackupStrategy(ctx context.Context, sql string) (*RecommendBackupStrategyRes, error)
+}
+
+type RecommendBackupStrategyRes struct {
+	BackupStrategy    string
+	BackupStrategyTip string
+	TablesRefer       []string
+	SchemasRefer      []string
 }
 
 type PluginProcessor interface {

sqle/driver/v2/driver_grpc_server.go

@@ -103,6 +103,14 @@ func (d *DriverGrpcServer) getDriverBySession(session *protoV2.Session) (Driver,
 		return nil, fmt.Errorf("session %s not found", session.Id)
 	}
 	return driver, nil
+
+}
+func (d *DriverGrpcServer) Backup(ctx context.Context, req *protoV2.BackupReq) (*protoV2.BackupRes, error) {
+	return nil, nil
+}
+
+func (d *DriverGrpcServer) RecommendBackupStrategy(ctx context.Context, req *protoV2.RecommendBackupStrategyReq) (*protoV2.RecommendBackupStrategyRes, error) {
+	return nil, nil
 }
 
 func (d *DriverGrpcServer) Metas(ctx context.Context, req *protoV2.Empty) (*protoV2.MetasResponse, error) {

sqle/driver/v2/driver_interface.go

@@ -93,6 +93,30 @@ type Driver interface {
 	KillProcess(ctx context.Context) (*KillProcessInfo, error)
 	GetDatabaseObjectDDL(ctx context.Context, objInfos []*DatabasSchemaInfo) ([]*DatabaseSchemaObjectResult, error)
 	GetDatabaseDiffModifySQL(ctx context.Context, calibratedDSN *DSN, objInfos []*DatabasCompareSchemaInfo) ([]*DatabaseDiffModifySQLResult, error)
+
+	Backup(ctx context.Context, req *BackupReq) (*BackupRes, error)
+	RecommendBackupStrategy(ctx context.Context, req *RecommendBackupStrategyReq) (*RecommendBackupStrategyRes, error)
+}
+
+type BackupReq struct {
+	BackupStrategy string
+	Sql            string
+}
+
+type BackupRes struct {
+	BackupSql   []string
+	ExecuteInfo string
+}
+
+type RecommendBackupStrategyReq struct {
+	Sql string
+}
+
+type RecommendBackupStrategyRes struct {
+	BackupStrategy    string
+	BackupStrategyTip string
+	TablesRefer       []string
+	SchemasRefer      []string
 }
 
 type Node struct {