Revert "feat(backend): dbbackup备份空间预测优化 close #1825"

This reverts commit f566d7c.

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_new_dbbackup.go

@@ -452,8 +452,6 @@ func (i *InstallNewDbBackupComp) ChownGroup() (err error) {
 	return nil
 }
 
-// saveTplConfigfile 渲染ini模板
-// todo: 将 Configs 转换成 struct，再把 struct 转换成 ini. 方便渲染 Public.EncryptOpt
 func (i *InstallNewDbBackupComp) saveTplConfigfile(tmpl string) (err error) {
 	f, err := os.OpenFile(tmpl, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0755)
 	if err != nil {
@@ -462,31 +460,20 @@ func (i *InstallNewDbBackupComp) saveTplConfigfile(tmpl string) (err error) {
 	defer func() {
 		_ = f.Close()
 	}()
-	var encryptOpt = make(map[string]string)
-	var encryptOptPrefix = "EncryptOpt"
-	for key, val := range i.Params.Configs {
-		_, err := fmt.Fprintf(f, "[%s]\n", key)
+
+	for k, v := range i.Params.Configs {
+		_, err := fmt.Fprintf(f, "[%s]\n", k)
 		if err != nil {
-			return errors.WithMessagef(err, "写配置模版 %s 失败", key)
+			return errors.WithMessagef(err, "写配置模版 %s 失败", k)
 		}
-		for k, v := range val {
-			if strings.HasPrefix(k, encryptOptPrefix+".") {
-				encryptOpt[strings.TrimPrefix(k, encryptOptPrefix+".")] = v
-				continue
-			}
+		for k, v := range v {
 			_, err := fmt.Fprintf(f, "%s = %s\n", k, v)
 			if err != nil {
 				return errors.WithMessagef(err, "写配置模版 %s, %s 失败", k, v)
 			}
 		}
 		fmt.Fprintf(f, "\n")
 	}
-	if len(encryptOpt) > 0 {
-		fmt.Fprintf(f, "[%s]\n", encryptOptPrefix)
-		for k, v := range encryptOpt {
-			fmt.Fprintf(f, "%s = %s\n", k, v)
-		}
-	}
 	return
 }
 

dbm-services/mysql/db-tools/mysql-dbbackup/Makefile

@@ -1,13 +1,12 @@
+DIST = $(error please set DIST flag to txsql or community)
 PROJ_BIN="dbbackup"
 PROJ="dbbackup-go"
 MODULE="dbm-services/mysql/db-tools/mysql-dbbackup"
 VERSION = $(error please set VERSION flag)
-DIST = $(error please set DIST flag to txsql or community)
 GITHASH = ""
 OUTPUT_DIR = build
 PKG_DIR=${OUTPUT_DIR}/${PROJ}
 PROJ_PKG=${PROJ}-${DIST}.tar.gz
-PROJ_PKG_DEPS=dbbackup-go-deps-${DIST}
 RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.gitHash=${GITHASH} "
 DEV_BUILD_FLAG = "-X ${MODULE}/cmd.version="develop" -X ${MODULE}/cmd.gitHash="" "
 BASE_DIR = $(shell pwd)
@@ -20,17 +19,16 @@ build:
 
 .PHONY: package
 package:build
-	echo "Packaging ${PROJ_PKG} using dependency ${PROJ_PKG_DEPS}"
+	echo "Packaging ${PROJ_PKG}"
 	mkdir -p ${PKG_DIR}
 	cp ${OUTPUT_DIR}/${PROJ_BIN} ${PKG_DIR}/
 	cp mydumper_for_tdbctl.cnf ${PKG_DIR}/
 	cp dbbackup_main.sh ${PKG_DIR}/
-	rm -rf ${PROJ_PKG_DEPS} && tar -zxf ${PROJ_PKG_DEPS}.tar.gz
-	cp -r ${PROJ_PKG_DEPS}/* ${PKG_DIR}/
+	cp -r dbbackup-go-deps/* ${PKG_DIR}/
 	chmod +x ${PKG_DIR}/*.sh && chmod +x ${PKG_DIR}/dbbackup
 	chmod +x ${PKG_DIR}/bin/* && chmod +x ${PKG_DIR}/bin/*/*
+
 	tar -C ${OUTPUT_DIR} -zcvf ${OUTPUT_DIR}/${PROJ_PKG} ${PROJ}
-	md5sum ${OUTPUT_DIR}/${PROJ_PKG}
 
 .PHONY: clean
 clean:

dbm-services/mysql/db-tools/mysql-dbbackup/build.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# release_type only used for release package name
+# we use dependency dir 'dbbackup-go-deps/' to tar, so make sure this dir has correct deps
+
+#### precheck section
+usage() {
+  echo "Usage:"
+  echo "./build.sh [-s] [-t your_release_type]"
+  echo -e "Description:\n"
+  echo "  -s: skip go build, use build/dbbackup binary"
+  echo "  -t: set release_type, allowed: txsql,community"
+  echo ""
+  exit 1
+}
+
+release_type=""
+go_build=1
+while getopts 't:sh' OPT; do
+    case $OPT in
+        t)
+          release_type="$OPTARG"
+          ;;
+        s)
+          go_build=0
+          ;;
+        h) usage;;
+        ?) usage;;
+    esac
+done
+if [ "$release_type" != "txsql" -a "$release_type" != "community" ];then
+  echo "unknown release_type. allowed: txsql,community"
+  exit 1
+else
+  echo "release_type=$release_type"
+fi
+
+#### build section
+proj_bin=dbbackup
+build_dir=build
+proj=dbbackup-go
+pkg_dir=${build_dir}/${proj}
+proj_pkg=${proj}-${release_type}.tar.gz
+
+if [ $go_build -ne 0 ];then
+  echo "run go build"
+  rm -rf $pkg_dir ; mkdir -p $pkg_dir
+  rm -f ${build_dir}/${proj_bin}
+  go build -o ${build_dir}/${proj_bin}
+  if [ $? -gt 0 ];then
+    echo "build dbbackup failed"
+    exit 1
+  fi
+else
+  echo "skip run go build. use binary build/dbbackup"
+  rm -rf $pkg_dir ; mkdir -p $pkg_dir
+  if [ ! -f ${build_dir}/${proj_bin} ];then
+    echo "${build_dir}/${proj_bin} not exists"
+    exit 1
+  fi
+fi
+
+cp -r dbbackup-go-deps/* ${pkg_dir}/
+if [ $? -gt 0 ];then
+  echo "copy dbbackup-go-deps failed"
+  exit 1
+fi
+
+cp -a dbbackup_main.sh ${pkg_dir}/
+cp -a mydumper_for_tdbctl.cnf ${pkg_dir}/
+cp -a ${build_dir}/${proj_bin} ${pkg_dir}/
+chmod +x ${pkg_dir}/*.sh && chmod +x ${pkg_dir}/dbbackup
+chmod +x ${pkg_dir}/bin/* && chmod +x ${pkg_dir}/bin/*/*
+
+tar -C ${build_dir} -zcvf ${build_dir}/${proj_pkg} ${proj}

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/config.go

@@ -9,3 +9,27 @@ type BackupConfig struct {
 	PhysicalBackup PhysicalBackup `ini:"PhysicalBackup"`
 	PhysicalLoad   PhysicalLoad   `ini:"PhysicalLoad"`
 }
+
+/*
+// LogicalBackupCnf Logical Backup BackupConfig
+type LogicalBackupCnf struct {
+	Public        Public        `ini:"Public" validate:"required"`
+	LogicalBackup LogicalBackup `ini:"LogicalBackup" validate:"required"`
+}
+
+// LogicalLoadCnf Logical Load BackupConfig
+type LogicalLoadCnf struct {
+	LogicalBackup LogicalBackup `ini:"LogicalBackup" validate:"required"`
+}
+
+// PhysicalBackupCnf Physical Backup BackupConfig
+type PhysicalBackupCnf struct {
+	Public         Public         `ini:"Public" validate:"required"`
+	PhysicalBackup PhysicalBackup `ini:"PhysicalBackup" validate:"required"`
+}
+
+// PhysicalLoadCnf Physical Load BackupConfig
+type PhysicalLoadCnf struct {
+	PhysicalLoad PhysicalLoad `ini:"PhysicalLoad" validate:"required"`
+}
+*/

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go

@@ -12,11 +12,11 @@ import (
 
 // Dumper TODO
 type Dumper interface {
-	initConfig(mysqlVersion string) error
+	initConfig() error
 	Execute(enableTimeOut bool) error
 }
 
-// BuildDumper return logical or physical dumper
+// BuildDumper TODO
 func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) {
 	if strings.ToLower(cnf.Public.BackupType) == "logical" {
 		if err := validate.GoValidateStruct(cnf.LogicalBackup, false, false); err != nil {
@@ -29,6 +29,7 @@ func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) {
 		if err := validate.GoValidateStruct(cnf.PhysicalBackup, false, false); err != nil {
 			return nil, err
 		}
+
 		dumper = &PhysicalDumper{
 			cnf: cnf,
 		}
@@ -37,5 +38,6 @@ func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) {
 		err := fmt.Errorf("unknown BackupType: %s", cnf.Public.BackupType)
 		return nil, err
 	}
+
 	return dumper, nil
 }

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go

@@ -21,7 +21,7 @@ type LogicalDumper struct {
 	dbbackupHome string
 }
 
-func (l *LogicalDumper) initConfig(mysqlVersion string) error {
+func (l *LogicalDumper) initConfig() error {
 	if l.cnf == nil {
 		return errors.New("logical dumper params is nil")
 	}
@@ -34,6 +34,82 @@ func (l *LogicalDumper) initConfig(mysqlVersion string) error {
 	return nil
 }
 
+// CreateDumpCmd Create LogicalBackup Cmd
+//func (l *LogicalDumper) CreateDumpCmd() (string, error) {
+//var buffer bytes.Buffer
+//binpath := filepath.Join(l.dbbackupHome, "/bin/mydumper")
+//buffer.WriteString(binpath)
+//buffer.WriteString(" -h " + l.cnf.Public.MysqlHost)
+//buffer.WriteString(" -P " + l.cnf.Public.MysqlPort)
+//buffer.WriteString(" -u " + l.cnf.Public.MysqlUser)
+//buffer.WriteString(" -p " + l.cnf.Public.MysqlPasswd)
+//buffer.WriteString(" -o " + l.cnf.Public.BackupDir + "/" + common.TargetName)
+//if !l.cnf.LogicalBackup.DisableCompress {
+//	buffer.WriteString(" --compress")
+//}
+//if l.cnf.LogicalBackup.DefaultsFile != "" {
+//	buffer.WriteString(" --defaults-file " + l.cnf.LogicalBackup.DefaultsFile)
+//}
+//buffer.WriteString(" --trx-consistency-only")
+//buffer.WriteString(" --long-query-retries " + cast.ToString(l.cnf.LogicalBackup.FlushRetryCount))
+//buffer.WriteString(" --set-names " + l.cnf.Public.MysqlCharset)
+//buffer.WriteString(" --chunk-filesize " + cast.ToString(l.cnf.LogicalBackup.ChunkFileSize))
+//buffer.WriteString(" --long-query-retry-interval 10")
+// -- buffer.WriteString(" --checksum-all")
+//if l.cnf.LogicalBackup.Regex != "" {
+//	buffer.WriteString(fmt.Sprintf(` -x '%s'`, l.cnf.LogicalBackup.Regex))
+//}
+//if common.BackupSchema && !common.BackupData { // backup schema only
+//	buffer.WriteString(" --no-data")
+//	buffer.WriteString(" --events --routines --triggers")
+//} else if !common.BackupSchema && common.BackupData { // backup data only
+//	buffer.WriteString(" --no-schemas --no-views")
+//} else if common.BackupSchema && common.BackupData { // all
+//	buffer.WriteString(" --events --routines --triggers")
+//}
+//buffer.WriteString(" --threads " + strconv.Itoa(l.cnf.LogicalBackup.Threads))
+//	buffer.WriteString(" " + l.cnf.LogicalBackup.ExtraOpt)
+//	buffer.WriteString(" > logs/mydumper_`date +%w`.log 2>&1")
+//
+//	cmdStr := buffer.String()
+//	logger.Log.Info(fmt.Sprintf("build backup cmd_line: %s", cmdStr))
+//	return cmdStr, nil
+//}
+
+//// Execute Execute logical dump command
+//func (l *LogicalDumper) Execute(enableTimeOut bool) error {
+//	cmdStr, err := l.CreateDumpCmd()
+//	if err != nil {
+//		logger.Log.Error("Failed to create the cmd_line of dumping backup, error: ", err)
+//		return err
+//	}
+//
+//	if enableTimeOut {
+//		timeDiffUnix, err := GetMaxRunningTime(l.cnf.Public.BackupTimeOut)
+//		if err != nil {
+//			return err
+//		}
+//		ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(timeDiffUnix))*time.Second)
+//		defer cancel()
+//
+//		// execute command with timeout
+//		res, exeErr := exec.CommandContext(ctx, "/bin/bash", "-c", cmdStr).CombinedOutput()
+//		logger.Log.Info("execute dumping logical backup with timeout, result:", string(res))
+//		if exeErr != nil {
+//			logger.Log.Error("Failed to execute dumping logical backup, error:", exeErr)
+//			return exeErr
+//		}
+//	} else {
+//		res, exeErr := exec.Command("/bin/bash", "-c", cmdStr).CombinedOutput()
+//		logger.Log.Info("execute dumping logical backup without timeout, result:", string(res))
+//		if exeErr != nil {
+//			logger.Log.Error("Failed to execute dumping logical backup, error:", exeErr)
+//			return exeErr
+//		}
+//	}
+//	return nil
+//}
+
 // Execute excute dumping backup with logical backup tool
 func (l *LogicalDumper) Execute(enableTimeOut bool) error {
 	binPath := filepath.Join(l.dbbackupHome, "/bin/mydumper")