feat(backend): dbbackup备份空间预测优化 close #1825

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_new_dbbackup.go

@@ -452,6 +452,8 @@ func (i *InstallNewDbBackupComp) ChownGroup() (err error) {
 	return nil
 }
 
+// saveTplConfigfile 渲染ini模板
+// todo: 将 Configs 转换成 struct，再把 struct 转换成 ini. 方便渲染 Public.EncryptOpt
 func (i *InstallNewDbBackupComp) saveTplConfigfile(tmpl string) (err error) {
 	f, err := os.OpenFile(tmpl, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0755)
 	if err != nil {
@@ -460,20 +462,31 @@ func (i *InstallNewDbBackupComp) saveTplConfigfile(tmpl string) (err error) {
 	defer func() {
 		_ = f.Close()
 	}()
-
-	for k, v := range i.Params.Configs {
-		_, err := fmt.Fprintf(f, "[%s]\n", k)
+	var encryptOpt = make(map[string]string)
+	var encryptOptPrefix = "EncryptOpt"
+	for key, val := range i.Params.Configs {
+		_, err := fmt.Fprintf(f, "[%s]\n", key)
 		if err != nil {
-			return errors.WithMessagef(err, "写配置模版 %s 失败", k)
+			return errors.WithMessagef(err, "写配置模版 %s 失败", key)
 		}
-		for k, v := range v {
+		for k, v := range val {
+			if strings.HasPrefix(k, encryptOptPrefix+".") {
+				encryptOpt[strings.TrimPrefix(k, encryptOptPrefix+".")] = v
+				continue
+			}
 			_, err := fmt.Fprintf(f, "%s = %s\n", k, v)
 			if err != nil {
 				return errors.WithMessagef(err, "写配置模版 %s, %s 失败", k, v)
 			}
 		}
 		fmt.Fprintf(f, "\n")
 	}
+	if len(encryptOpt) > 0 {
+		fmt.Fprintf(f, "[%s]\n", encryptOptPrefix)
+		for k, v := range encryptOpt {
+			fmt.Fprintf(f, "%s = %s\n", k, v)
+		}
+	}
 	return
 }
 

dbm-services/mysql/db-tools/mysql-dbbackup/Makefile

@@ -1,12 +1,13 @@
-DIST = $(error please set DIST flag to txsql or community)
 PROJ_BIN="dbbackup"
 PROJ="dbbackup-go"
 MODULE="dbm-services/mysql/db-tools/mysql-dbbackup"
 VERSION = $(error please set VERSION flag)
+DIST = $(error please set DIST flag to txsql or community)
 GITHASH = ""
 OUTPUT_DIR = build
 PKG_DIR=${OUTPUT_DIR}/${PROJ}
 PROJ_PKG=${PROJ}-${DIST}.tar.gz
+PROJ_PKG_DEPS=dbbackup-go-deps-${DIST}
 RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.gitHash=${GITHASH} "
 DEV_BUILD_FLAG = "-X ${MODULE}/cmd.version="develop" -X ${MODULE}/cmd.gitHash="" "
 BASE_DIR = $(shell pwd)
@@ -19,16 +20,17 @@ build:
 
 .PHONY: package
 package:build
-	echo "Packaging ${PROJ_PKG}"
+	echo "Packaging ${PROJ_PKG} using dependency ${PROJ_PKG_DEPS}"
 	mkdir -p ${PKG_DIR}
 	cp ${OUTPUT_DIR}/${PROJ_BIN} ${PKG_DIR}/
 	cp mydumper_for_tdbctl.cnf ${PKG_DIR}/
 	cp dbbackup_main.sh ${PKG_DIR}/
-	cp -r dbbackup-go-deps/* ${PKG_DIR}/
+	rm -rf ${PROJ_PKG_DEPS} && tar -zxf ${PROJ_PKG_DEPS}.tar.gz
+	cp -r ${PROJ_PKG_DEPS}/* ${PKG_DIR}/
 	chmod +x ${PKG_DIR}/*.sh && chmod +x ${PKG_DIR}/dbbackup
 	chmod +x ${PKG_DIR}/bin/* && chmod +x ${PKG_DIR}/bin/*/*
-
 	tar -C ${OUTPUT_DIR} -zcvf ${OUTPUT_DIR}/${PROJ_PKG} ${PROJ}
+	md5sum ${OUTPUT_DIR}/${PROJ_PKG}
 
 .PHONY: clean
 clean:

dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_version.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// versionCmd represents the version command
+var subCmdVersion = &cobra.Command{
+	Use:   "version",
+	Short: "version information",
+	Long:  `version information about buildStamp, gitHash`,
+	Run: func(cmd *cobra.Command, args []string) {
+		printVersion()
+	},
+}
+var version = ""
+var buildStamp = ""
+var gitHash = ""
+
+func init() {
+	rootCmd.AddCommand(subCmdVersion)
+}
+func printVersion() {
+	fmt.Printf("Version: %s, GitHash: %s, BuildAt: %s\n", version, gitHash, buildStamp)
+}

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/config.go

@@ -9,27 +9,3 @@ type BackupConfig struct {
 	PhysicalBackup PhysicalBackup `ini:"PhysicalBackup"`
 	PhysicalLoad   PhysicalLoad   `ini:"PhysicalLoad"`
 }
-
-/*
-// LogicalBackupCnf Logical Backup BackupConfig
-type LogicalBackupCnf struct {
-	Public        Public        `ini:"Public" validate:"required"`
-	LogicalBackup LogicalBackup `ini:"LogicalBackup" validate:"required"`
-}
-
-// LogicalLoadCnf Logical Load BackupConfig
-type LogicalLoadCnf struct {
-	LogicalBackup LogicalBackup `ini:"LogicalBackup" validate:"required"`
-}
-
-// PhysicalBackupCnf Physical Backup BackupConfig
-type PhysicalBackupCnf struct {
-	Public         Public         `ini:"Public" validate:"required"`
-	PhysicalBackup PhysicalBackup `ini:"PhysicalBackup" validate:"required"`
-}
-
-// PhysicalLoadCnf Physical Load BackupConfig
-type PhysicalLoadCnf struct {
-	PhysicalLoad PhysicalLoad `ini:"PhysicalLoad" validate:"required"`
-}
-*/

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go

@@ -12,11 +12,11 @@ import (
 
 // Dumper TODO
 type Dumper interface {
-	initConfig() error
+	initConfig(mysqlVersion string) error
 	Execute(enableTimeOut bool) error
 }
 
-// BuildDumper TODO
+// BuildDumper return logical or physical dumper
 func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) {
 	if strings.ToLower(cnf.Public.BackupType) == "logical" {
 		if err := validate.GoValidateStruct(cnf.LogicalBackup, false, false); err != nil {
@@ -29,7 +29,6 @@ func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) {
 		if err := validate.GoValidateStruct(cnf.PhysicalBackup, false, false); err != nil {
 			return nil, err
 		}
-
 		dumper = &PhysicalDumper{
 			cnf: cnf,
 		}
@@ -38,6 +37,5 @@ func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) {
 		err := fmt.Errorf("unknown BackupType: %s", cnf.Public.BackupType)
 		return nil, err
 	}
-
 	return dumper, nil
 }

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go

@@ -21,7 +21,7 @@ type LogicalDumper struct {
 	dbbackupHome string
 }
 
-func (l *LogicalDumper) initConfig() error {
+func (l *LogicalDumper) initConfig(mysqlVersion string) error {
 	if l.cnf == nil {
 		return errors.New("logical dumper params is nil")
 	}
@@ -34,82 +34,6 @@ func (l *LogicalDumper) initConfig() error {
 	return nil
 }
 
-// CreateDumpCmd Create LogicalBackup Cmd
-//func (l *LogicalDumper) CreateDumpCmd() (string, error) {
-//var buffer bytes.Buffer
-//binpath := filepath.Join(l.dbbackupHome, "/bin/mydumper")
-//buffer.WriteString(binpath)
-//buffer.WriteString(" -h " + l.cnf.Public.MysqlHost)
-//buffer.WriteString(" -P " + l.cnf.Public.MysqlPort)
-//buffer.WriteString(" -u " + l.cnf.Public.MysqlUser)
-//buffer.WriteString(" -p " + l.cnf.Public.MysqlPasswd)
-//buffer.WriteString(" -o " + l.cnf.Public.BackupDir + "/" + common.TargetName)
-//if !l.cnf.LogicalBackup.DisableCompress {
-//	buffer.WriteString(" --compress")
-//}
-//if l.cnf.LogicalBackup.DefaultsFile != "" {
-//	buffer.WriteString(" --defaults-file " + l.cnf.LogicalBackup.DefaultsFile)
-//}
-//buffer.WriteString(" --trx-consistency-only")
-//buffer.WriteString(" --long-query-retries " + cast.ToString(l.cnf.LogicalBackup.FlushRetryCount))
-//buffer.WriteString(" --set-names " + l.cnf.Public.MysqlCharset)
-//buffer.WriteString(" --chunk-filesize " + cast.ToString(l.cnf.LogicalBackup.ChunkFileSize))
-//buffer.WriteString(" --long-query-retry-interval 10")
-// -- buffer.WriteString(" --checksum-all")
-//if l.cnf.LogicalBackup.Regex != "" {
-//	buffer.WriteString(fmt.Sprintf(` -x '%s'`, l.cnf.LogicalBackup.Regex))
-//}
-//if common.BackupSchema && !common.BackupData { // backup schema only
-//	buffer.WriteString(" --no-data")
-//	buffer.WriteString(" --events --routines --triggers")
-//} else if !common.BackupSchema && common.BackupData { // backup data only
-//	buffer.WriteString(" --no-schemas --no-views")
-//} else if common.BackupSchema && common.BackupData { // all
-//	buffer.WriteString(" --events --routines --triggers")
-//}
-//buffer.WriteString(" --threads " + strconv.Itoa(l.cnf.LogicalBackup.Threads))
-//	buffer.WriteString(" " + l.cnf.LogicalBackup.ExtraOpt)
-//	buffer.WriteString(" > logs/mydumper_`date +%w`.log 2>&1")
-//
-//	cmdStr := buffer.String()
-//	logger.Log.Info(fmt.Sprintf("build backup cmd_line: %s", cmdStr))
-//	return cmdStr, nil
-//}
-
-//// Execute Execute logical dump command
-//func (l *LogicalDumper) Execute(enableTimeOut bool) error {
-//	cmdStr, err := l.CreateDumpCmd()
-//	if err != nil {
-//		logger.Log.Error("Failed to create the cmd_line of dumping backup, error: ", err)
-//		return err
-//	}
-//
-//	if enableTimeOut {
-//		timeDiffUnix, err := GetMaxRunningTime(l.cnf.Public.BackupTimeOut)
-//		if err != nil {
-//			return err
-//		}
-//		ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(timeDiffUnix))*time.Second)
-//		defer cancel()
-//
-//		// execute command with timeout
-//		res, exeErr := exec.CommandContext(ctx, "/bin/bash", "-c", cmdStr).CombinedOutput()
-//		logger.Log.Info("execute dumping logical backup with timeout, result:", string(res))
-//		if exeErr != nil {
-//			logger.Log.Error("Failed to execute dumping logical backup, error:", exeErr)
-//			return exeErr
-//		}
-//	} else {
-//		res, exeErr := exec.Command("/bin/bash", "-c", cmdStr).CombinedOutput()
-//		logger.Log.Info("execute dumping logical backup without timeout, result:", string(res))
-//		if exeErr != nil {
-//			logger.Log.Error("Failed to execute dumping logical backup, error:", exeErr)
-//			return exeErr
-//		}
-//	}
-//	return nil
-//}
-
 // Execute excute dumping backup with logical backup tool
 func (l *LogicalDumper) Execute(enableTimeOut bool) error {
 	binPath := filepath.Join(l.dbbackupHome, "/bin/mydumper")