fix(mysql): 闪回修复时间格式，mydumper增加lock-wait-timeout TencentBlueKing#8632

dbm-services/common/go-pubpkg/cmutil/time.go

@@ -1,10 +1,26 @@
 package cmutil
 
-import "time"
+import (
+	"time"
+
+	"github.com/pkg/errors"
+)
 
 // TimeToSecondPrecision keep only second precision for time
 func TimeToSecondPrecision(t time.Time) time.Time {
 	timeStr := t.Local().Format(time.RFC3339)
 	tt, _ := time.ParseInLocation(time.RFC3339, timeStr, time.Local)
 	return tt
 }
+
+// ParseLocalTimeString 讲 time.Datetime or time.RFC3339 格式转换传本地时区 time.Time 类型
+func ParseLocalTimeString(s string) (time.Time, error) {
+	t, err := time.ParseInLocation(time.DateTime, s, time.Local)
+	if err != nil {
+		if t, err = time.ParseInLocation(time.RFC3339, s, time.Local); err != nil {
+			return time.Time{},
+				errors.Errorf("expect time format '%s' or '%s' but got '%s'", time.DateTime, time.RFC3339, s)
+		}
+	}
+	return t, nil
+}

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/goapply_binlog.go

@@ -535,18 +535,19 @@ func (r *GoApplyBinlog) FilterBinlogFiles() (totalSize int64, err error) {
 	var firstBinlogSize int64 = 0
 	// 过滤 binlog time < stop_time
 	// 如果有需要 也会过滤 binlog time > start_time
-	var startTimeMore, stopTimeMore time.Time // 前后时间偏移 20分钟
-	var startTimeFilter, stopTimeFilter string
+	var startTimeFilter, stopTimeFilter time.Time // 前后时间偏移 20分钟
 	if r.BinlogOpt.StartTime != "" {
-		startTimeMore, _ = time.ParseInLocation(time.DateTime, r.BinlogOpt.StartTime, time.Local)
+		if startTimeFilter, err = cmutil.ParseLocalTimeString(r.BinlogOpt.StartTime); err != nil {
+			return 0, errors.WithMessage(err, "start_time parse failed")
+		}
 		// binlog时间 start_time 比 预期start_time 提早 20 分钟
-		startTimeFilter = startTimeMore.Add(-20 * time.Minute).Format(time.RFC3339)
+		startTimeFilter = startTimeFilter.Add(-20 * time.Minute)
 	}
-	if stopTimeMore, err = time.ParseInLocation(time.DateTime, r.BinlogOpt.StopTime, time.Local); err != nil {
-		return 0, errors.Errorf("stop_time parse failed: %s", r.BinlogOpt.StopTime)
+	if stopTimeFilter, err = cmutil.ParseLocalTimeString(r.BinlogOpt.StopTime); err != nil {
+		return 0, errors.WithMessage(err, "stop_time parse failed")
 	} else {
 		// binlog时间 stop_time 比 预期stop_time 延后 20 分钟
-		stopTimeFilter = stopTimeMore.Add(20 * time.Minute).Format(time.RFC3339)
+		stopTimeFilter = stopTimeFilter.Add(20 * time.Minute)
 	}
 
 	for _, f := range r.BinlogFiles {
@@ -557,19 +558,19 @@ func (r *GoApplyBinlog) FilterBinlogFiles() (totalSize int64, err error) {
 		if err != nil {
 			return 0, err
 		}
-		startTime := events[0].EventTime
-		stopTime := events[1].EventTime
+		startTime, _ := time.ParseInLocation(time.RFC3339, events[0].EventTime, time.Local)
+		stopTime, _ := time.ParseInLocation(time.RFC3339, events[1].EventTime, time.Local)
 		fileSize := cmutil.GetFileSize(fileName)
 		// **** get binlog time
 
-		if r.BinlogOpt.StopTime != "" && stopTime > stopTimeFilter {
+		if r.BinlogOpt.StopTime != "" && stopTime.Compare(stopTimeFilter) > 0 {
 			break
 		}
 		if r.BinlogStartFile != "" {
 			binlogFiles = append(binlogFiles, f)
 			totalSize += fileSize
 		} else if r.BinlogOpt.StartTime != "" {
-			if startTime > startTimeFilter { // time.RFC3339
+			if startTime.Compare(startTimeFilter) > 0 { // time.RFC3339
 				if !firstBinlogFound { // 拿到binlog时间符合条件的 前一个binlog
 					firstBinlogFound = true
 					firstBinlogFile = lastBinlogFile

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/gomysqlbinlog_util.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cast"
 
+	"dbm-services/common/go-pubpkg/cmutil"
 	"dbm-services/common/go-pubpkg/logger"
 	"dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil"
 )
@@ -26,10 +27,8 @@ type GoMySQLBinlogUtil struct {
 	BinlogDir string `json:"-"`
 	StartFile string `json:"-"`
 	StopFile  string `json:"-"`
-	// --start-datetime  时间格式
-	// 格式 "2006-01-02 15:04:05" 原样传递给 mysqlbinlog
-	// 格式"2006-01-02T15:04:05Z07:00"(示例"2023-12-11T05:03:05+08:00")按照机器本地时间，解析成 "2006-01-02 15:04:05" 再传递给 mysqlbinlog
-	// 在 Init 时会统一把时间字符串转换成 time.RFC3399
+	// --start-datetime  时间格式，支持 "2006-01-02 15:04:05" or "2006-01-02T15:04:05Z07:00" 两种格式
+	// 传递给 gomysqlbinlog 时，会按照机器本地时间，统一解析成 "2006-01-02 15:04:05"
 	StartTime string `json:"start_time"`
 	// --stop-datetime   时间格式同 StartTime，可带时区，会转换成机器本地时间
 	StopTime string `json:"stop_time"`
@@ -113,17 +112,19 @@ func (b *GoMySQLBinlogUtil) BuildArgs(filterMode bool) ([]string, error) {
 		}
 	}
 	if b.StartTime != "" {
-		_, err := time.ParseInLocation(time.DateTime, b.StartTime, time.Local)
+		t, err := cmutil.ParseLocalTimeString(b.StartTime)
 		if err != nil {
-			return nil, errors.Errorf("start_time expect format %s but got %s", time.DateTime, b.StartTime)
+			return nil, errors.WithMessage(err, "start_time error")
 		}
+		b.StartTime = t.Format(time.DateTime)
 		b.cmdArgs = append(b.cmdArgs, fmt.Sprintf("--start-datetime='%s'", b.StartTime))
 	}
 	if b.StopTime != "" {
-		_, err := time.ParseInLocation(time.DateTime, b.StopTime, time.Local)
+		t, err := cmutil.ParseLocalTimeString(b.StopTime)
 		if err != nil {
-			return nil, errors.Errorf("stop_time expect format %s but got %s", time.DateTime, b.StopTime)
+			return nil, errors.WithMessage(err, "stop_time error")
 		}
+		b.StopTime = t.Format(time.DateTime)
 		b.cmdArgs = append(b.cmdArgs, fmt.Sprintf("--stop-datetime='%s'", b.StopTime))
 	} else {
 		return nil, errors.Errorf("stop-datetime cannot be empty")

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/rollback/goflashback_comp.go

@@ -63,7 +63,6 @@ func (f *GoFlashback) Init() error {
 		WorkDir:            f.WorkDir,
 		WorkID:             f.WorkID,
 		ParseConcurrency:   f.ParseConcurrency,
-		StartTime:          f.TargetTime,
 		QuickMode:          true,
 		SourceBinlogFormat: "ROW", // 这里只代表 flashback 要求 ROW 模式，源实例 binlog_format 在 PreCheck 里会判断
 		ParseOnly:          true,
@@ -89,17 +88,17 @@ func (f *GoFlashback) Init() error {
 	if f.StopTime == "" {
 		timeNow := time.Now()
 		f.StopTime = timeNow.Local().Format(time.RFC3339)
-		f.flashback.StopTime = f.StopTime
-		//f.flashback.BinlogOpt.StopTime = timeNow.Local().Format(time.DateTime)
 	}
+	f.flashback.StopTime = f.StopTime
+
 	toolset, err := tools.NewToolSetWithPick(tools.ToolGoMysqlbinlog, tools.ToolMysqlclient)
 	if err != nil {
 		return err
 	}
 	if err = f.flashback.ToolSet.Merge(toolset); err != nil {
 		return err
 	}
-	// 拼接 recover-binlog 参数
+	// 拼接 goapply-binlog 参数
 	if err := f.flashback.Init(); err != nil {
 		return err
 	}
@@ -140,6 +139,9 @@ func (f *GoFlashback) getBinlogFilesLocal() (int64, error) {
 		if err != nil {
 			return 0, err
 		}
+		if len(binlogFiles) == 0 {
+			logger.Warn("no binlog files found from %s", binlogDir)
+		}
 		f.flashback.BinlogDir = binlogDir // 实例真实 binlog dir
 		f.flashback.BinlogFiles = binlogFiles
 	}

dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md

@@ -323,40 +323,39 @@ DefaultsFile = /etc/my.cnf.3306
 
 ## 参数解释
 ### Public
-- Public.KillLongQueryTime
- 这个参数对逻辑备份和物理备份作用不同
- - 逻辑备份 mydumper
+- Public.KillLongQueryTime  
+ 这个参数对逻辑备份和物理备份作用不同，需要备份账号有 super 权限。默认为 0 则不 kill。
+ - 逻辑备份 mydumper  
   相当于`--kill-long-queries --long-query-guard xx`: 发出 FTWRL 之前如果发现有超过这个时间的长 sql，则 kill 掉
- - 物理备份 xtrabackup
+ - 物理备份 xtrabackup  
    相当于`--kill-long-queries-timeout=xx`: 发出 FTWRL 之后如果被阻塞，则等待多久之后把引起阻塞的长 sql kill 掉
 
-需要备份账号有 super 权限。默认为 0 则不 kill。
-
-- Public.FtwrlWaitTimeout
-  发起备份前检查长 sql，(如果不自动 kill/ kill失败) 则等待长 sql 多久后，放弃 ftwrl，放弃备份。
-  此时还未发起 `FLUSH TABLE WITH READ LOCK` 命令
-  默认 120s，对 mydumper / xtrabackup 有效
-  - 逻辑备份 mydumper
+- Public.FtwrlWaitTimeout  
+  发起备份前检查长 sql，(如果不自动 kill/ kill失败) 则等待长 sql 多久后，放弃 ftwrl，放弃备份。  
+  此时还未发起 `FLUSH TABLE WITH READ LOCK` 命令。默认 120s，对 mydumper / xtrabackup 有效
+  - 逻辑备份 mydumper  
     相当于 `--long-query-guard xx` 且不 kill
-  - 物理备份 xtrabackup
+  - 物理备份 xtrabackup  
     5.5, 5.6 : `--lock-wait-timeout`
     5.7, 8.0 : `--ftwrl-wait-timeout`
 
-- Public.AcquireLockWaitTimeout
-  备份加锁超时，比如 `LOCK TABLES FOR BACKUP` / `FLUSH TABLE WITH READ LOCK`，相当于 `set session lock-wait-timeout=xxx`
+- Public.AcquireLockWaitTimeout  
+  备份加锁超时，比如 `LOCK TABLES FOR BACKUP` / `FLUSH TABLE WITH READ LOCK`，相当于 `set session lock-wait-timeout=xxx`  
   默认 10s 超时，对 mydumper / xtrabackup 有效
 
-- Public.BackupTimeOut
-  备份超时结束时间，只对 master 有效，用于保护 master 不被备份影响。
+- Public.BackupTimeOut  
+  备份超时结束时间，只对 master 有效，用于保护 master 不被备份影响。  
   默认`09:00:00`，即备份执行到这个时间点还未结束，则退出
 
-- Public.OldFileLeftDay
+- Public.OldFileLeftDay  
   备份文件本地最大保留时间天数，每次备份之前会先清理旧的备份。如果备份空间不足，可能会继续清理备份文件，优先保证备份成功
 
-- Public.IOLimitMasterFactor
+- Public.IOLimitMasterFactor  
   master机器专用限速因子，因为备份速度可能有多个选项来控制
   - master 文件io打包限速: `Public.IOLimitMBPerSec * IOLimitMasterFactor`
   - 物理备份限速: `PhysicalBackup.Throttle * IOLimitMasterFactor`
 
-### LogicalBackup
-- LogicalBackup.TrxConsistencyOnly
+### LogicalBackup  
+- LogicalBackup.TrxConsistencyOnly  
+  mydumper `--trx-consistency-only`, 或者 mysqldump `--single-transaction`。默认 true  
+  对于多引擎混合的实例，如果想要保证整体数据的全局一致，需要设置为 false，会导致在整个备份期间持有 FTWRL，在主库上谨慎使用false。

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go

@@ -243,6 +243,7 @@ func (l *LogicalDumper) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbarep
 // MydumperHasOption check mydumper has --xxx or not
 // example: ./mydumper --lock-wait-timeout 1 --help
 func MydumperHasOption(bin string, option ...string) (bool, error) {
+	// --help 在前/后 无所谓
 	cmdArgs := []string{bin, "--help"}
 	cmdArgs = append(cmdArgs, option...)
 	_, cmdStderr, err := cmutil.ExecCommand(false, "", cmdArgs[0], cmdArgs[1:]...)

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go

@@ -160,6 +160,7 @@ func (l *LogicalDumperMysqldump) buildArgsObjectFilter() (args []string) {
 
 // MysqldumpHasOption check mysqldump has --xxx or not
 func MysqldumpHasOption(bin string, option string) (bool, error) {
+	// 注意 --help 要在后面
 	_, cmdStderr, err := cmutil.ExecCommand(false, "", bin, option, "--help")
 	if err == nil {
 		return true, nil

dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/binlog-parser/parse_binlog_time.go

@@ -314,6 +314,6 @@ func (b *BinlogParse) GetRotateEvent(f *os.File) (*replication.EventHeader, erro
 			}
 		}
 	}
-	// if mysqld is shutdown with kill -9, binlog maynot contains rotate/stop event
+	// if mysqld is shutdown with kill -9, binlog may not contains rotate/stop event
 	return nil, errors.Errorf("%s: get RotateEvent or StopEvent failed", b.FileName)
 }