From a579edb7ec3b5fd05db32a1b6e2489305525c19c Mon Sep 17 00:00:00 2001 From: seanlook Date: Thu, 16 Nov 2023 12:26:04 +0800 Subject: [PATCH] =?UTF-8?q?feat(backend):=20dbbackup=E5=A4=87=E4=BB=BD?= =?UTF-8?q?=E7=A9=BA=E9=97=B4=E9=A2=84=E6=B5=8B=E4=BC=98=E5=8C=96=20close?= =?UTF-8?q?=20#1825?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/mysql/install_new_dbbackup.go | 23 ++++- .../mysql/db-tools/mysql-dbbackup/Makefile | 10 +- .../mysql/db-tools/mysql-dbbackup/build.sh | 75 --------------- .../mysql-dbbackup/cmd/subcmd_version.go | 27 ++++++ .../mysql-dbbackup/pkg/config/config.go | 24 ----- .../pkg/src/backupexe/dumper.go | 6 +- .../pkg/src/backupexe/dumper_logical.go | 78 +--------------- .../pkg/src/backupexe/dumper_physical.go | 91 +------------------ .../mysql-dbbackup/pkg/src/backupexe/env.go | 10 +- .../pkg/src/backupexe/execute_dump.go | 15 +-- .../db-tools/mysql-dbbackup/pkg/util/misc.go | 1 - .../mysql-dbbackup/release_package.sh | 10 +- .../mysql-rotatebinlog/cmd/subcmd_version.go | 27 ++++++ dbm-ui/backend/dbm_init/medium/Dockerfile | 9 +- dbm-ui/backend/dbm_init/medium/medium.lock | 4 +- 15 files changed, 113 insertions(+), 297 deletions(-) delete mode 100755 dbm-services/mysql/db-tools/mysql-dbbackup/build.sh create mode 100644 dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_version.go create mode 100644 dbm-services/mysql/db-tools/mysql-rotatebinlog/cmd/subcmd_version.go diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_new_dbbackup.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_new_dbbackup.go index 004bc12ce2..fb74584755 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_new_dbbackup.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_new_dbbackup.go @@ -452,6 +452,8 @@ func (i *InstallNewDbBackupComp) ChownGroup() (err error) { return nil } +// saveTplConfigfile 渲染ini模板 +// todo: 将 Configs 转换成 struct，再把 struct 转换成 ini. 方便渲染 Public.EncryptOpt func (i *InstallNewDbBackupComp) saveTplConfigfile(tmpl string) (err error) { f, err := os.OpenFile(tmpl, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0755) if err != nil { @@ -460,13 +462,18 @@ func (i *InstallNewDbBackupComp) saveTplConfigfile(tmpl string) (err error) { defer func() { _ = f.Close() }() - - for k, v := range i.Params.Configs { - _, err := fmt.Fprintf(f, "[%s]\n", k) + var encryptOpt = make(map[string]string) + var encryptOptPrefix = "EncryptOpt" + for key, val := range i.Params.Configs { + _, err := fmt.Fprintf(f, "[%s]\n", key) if err != nil { - return errors.WithMessagef(err, "写配置模版 %s 失败", k) + return errors.WithMessagef(err, "写配置模版 %s 失败", key) } - for k, v := range v { + for k, v := range val { + if strings.HasPrefix(k, encryptOptPrefix+".") { + encryptOpt[strings.TrimPrefix(k, encryptOptPrefix+".")] = v + continue + } _, err := fmt.Fprintf(f, "%s = %s\n", k, v) if err != nil { return errors.WithMessagef(err, "写配置模版 %s, %s 失败", k, v) @@ -474,6 +481,12 @@ func (i *InstallNewDbBackupComp) saveTplConfigfile(tmpl string) (err error) { } fmt.Fprintf(f, "\n") } + if len(encryptOpt) > 0 { + fmt.Fprintf(f, "[%s]\n", encryptOptPrefix) + for k, v := range encryptOpt { + fmt.Fprintf(f, "%s = %s\n", k, v) + } + } return } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/Makefile b/dbm-services/mysql/db-tools/mysql-dbbackup/Makefile index ee4af256aa..e4612ce212 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/Makefile +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/Makefile @@ -1,12 +1,13 @@ -DIST = $(error please set DIST flag to txsql or community) PROJ_BIN="dbbackup" PROJ="dbbackup-go" MODULE="dbm-services/mysql/db-tools/mysql-dbbackup" VERSION = $(error please set VERSION flag) +DIST = $(error please set DIST flag to txsql or community) GITHASH = "" OUTPUT_DIR = build PKG_DIR=${OUTPUT_DIR}/${PROJ} PROJ_PKG=${PROJ}-${DIST}.tar.gz +PROJ_PKG_DEPS=dbbackup-go-deps-${DIST} RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.gitHash=${GITHASH} " DEV_BUILD_FLAG = "-X ${MODULE}/cmd.version="develop" -X ${MODULE}/cmd.gitHash="" " BASE_DIR = $(shell pwd) @@ -19,16 +20,17 @@ build: .PHONY: package package:build - echo "Packaging ${PROJ_PKG}" + echo "Packaging ${PROJ_PKG} using dependency ${PROJ_PKG_DEPS}" mkdir -p ${PKG_DIR} cp ${OUTPUT_DIR}/${PROJ_BIN} ${PKG_DIR}/ cp mydumper_for_tdbctl.cnf ${PKG_DIR}/ cp dbbackup_main.sh ${PKG_DIR}/ - cp -r dbbackup-go-deps/* ${PKG_DIR}/ + rm -rf ${PROJ_PKG_DEPS} && tar -zxf ${PROJ_PKG_DEPS}.tar.gz + cp -r ${PROJ_PKG_DEPS}/* ${PKG_DIR}/ chmod +x ${PKG_DIR}/*.sh && chmod +x ${PKG_DIR}/dbbackup chmod +x ${PKG_DIR}/bin/* && chmod +x ${PKG_DIR}/bin/*/* - tar -C ${OUTPUT_DIR} -zcvf ${OUTPUT_DIR}/${PROJ_PKG} ${PROJ} + md5sum ${OUTPUT_DIR}/${PROJ_PKG} .PHONY: clean clean: diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/build.sh b/dbm-services/mysql/db-tools/mysql-dbbackup/build.sh deleted file mode 100755 index d813c32aa8..0000000000 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/build.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -# release_type only used for release package name -# we use dependency dir 'dbbackup-go-deps/' to tar, so make sure this dir has correct deps - -#### precheck section -usage() { - echo "Usage:" - echo "./build.sh [-s] [-t your_release_type]" - echo -e "Description:\n" - echo " -s: skip go build, use build/dbbackup binary" - echo " -t: set release_type, allowed: txsql,community" - echo "" - exit 1 -} - -release_type="" -go_build=1 -while getopts 't:sh' OPT; do - case $OPT in - t) - release_type="$OPTARG" - ;; - s) - go_build=0 - ;; - h) usage;; - ?) usage;; - esac -done -if [ "$release_type" != "txsql" -a "$release_type" != "community" ];then - echo "unknown release_type. allowed: txsql,community" - exit 1 -else - echo "release_type=$release_type" -fi - -#### build section -proj_bin=dbbackup -build_dir=build -proj=dbbackup-go -pkg_dir=${build_dir}/${proj} -proj_pkg=${proj}-${release_type}.tar.gz - -if [ $go_build -ne 0 ];then - echo "run go build" - rm -rf $pkg_dir ; mkdir -p $pkg_dir - rm -f ${build_dir}/${proj_bin} - go build -o ${build_dir}/${proj_bin} - if [ $? -gt 0 ];then - echo "build dbbackup failed" - exit 1 - fi -else - echo "skip run go build. use binary build/dbbackup" - rm -rf $pkg_dir ; mkdir -p $pkg_dir - if [ ! -f ${build_dir}/${proj_bin} ];then - echo "${build_dir}/${proj_bin} not exists" - exit 1 - fi -fi - -cp -r dbbackup-go-deps/* ${pkg_dir}/ -if [ $? -gt 0 ];then - echo "copy dbbackup-go-deps failed" - exit 1 -fi - -cp -a dbbackup_main.sh ${pkg_dir}/ -cp -a mydumper_for_tdbctl.cnf ${pkg_dir}/ -cp -a ${build_dir}/${proj_bin} ${pkg_dir}/ -chmod +x ${pkg_dir}/*.sh && chmod +x ${pkg_dir}/dbbackup -chmod +x ${pkg_dir}/bin/* && chmod +x ${pkg_dir}/bin/*/* - -tar -C ${build_dir} -zcvf ${build_dir}/${proj_pkg} ${proj} diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_version.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_version.go new file mode 100644 index 0000000000..b05baf5a24 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_version.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// versionCmd represents the version command +var subCmdVersion = &cobra.Command{ + Use: "version", + Short: "version information", + Long: `version information about buildStamp, gitHash`, + Run: func(cmd *cobra.Command, args []string) { + printVersion() + }, +} +var version = "" +var buildStamp = "" +var gitHash = "" + +func init() { + rootCmd.AddCommand(subCmdVersion) +} +func printVersion() { + fmt.Printf("Version: %s, GitHash: %s, BuildAt: %s\n", version, gitHash, buildStamp) +} diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/config.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/config.go index c61d08eea1..5ae926257f 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/config.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/config.go @@ -9,27 +9,3 @@ type BackupConfig struct { PhysicalBackup PhysicalBackup `ini:"PhysicalBackup"` PhysicalLoad PhysicalLoad `ini:"PhysicalLoad"` } - -/* -// LogicalBackupCnf Logical Backup BackupConfig -type LogicalBackupCnf struct { - Public Public `ini:"Public" validate:"required"` - LogicalBackup LogicalBackup `ini:"LogicalBackup" validate:"required"` -} - -// LogicalLoadCnf Logical Load BackupConfig -type LogicalLoadCnf struct { - LogicalBackup LogicalBackup `ini:"LogicalBackup" validate:"required"` -} - -// PhysicalBackupCnf Physical Backup BackupConfig -type PhysicalBackupCnf struct { - Public Public `ini:"Public" validate:"required"` - PhysicalBackup PhysicalBackup `ini:"PhysicalBackup" validate:"required"` -} - -// PhysicalLoadCnf Physical Load BackupConfig -type PhysicalLoadCnf struct { - PhysicalLoad PhysicalLoad `ini:"PhysicalLoad" validate:"required"` -} -*/ diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go index 3c1451156b..c371226d2f 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go @@ -12,11 +12,11 @@ import ( // Dumper TODO type Dumper interface { - initConfig() error + initConfig(mysqlVersion string) error Execute(enableTimeOut bool) error } -// BuildDumper TODO +// BuildDumper return logical or physical dumper func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) { if strings.ToLower(cnf.Public.BackupType) == "logical" { if err := validate.GoValidateStruct(cnf.LogicalBackup, false, false); err != nil { @@ -29,7 +29,6 @@ func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) { if err := validate.GoValidateStruct(cnf.PhysicalBackup, false, false); err != nil { return nil, err } - dumper = &PhysicalDumper{ cnf: cnf, } @@ -38,6 +37,5 @@ func BuildDumper(cnf *config.BackupConfig) (dumper Dumper, err error) { err := fmt.Errorf("unknown BackupType: %s", cnf.Public.BackupType) return nil, err } - return dumper, nil } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go index 63505b0155..c4162da9c9 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go @@ -21,7 +21,7 @@ type LogicalDumper struct { dbbackupHome string } -func (l *LogicalDumper) initConfig() error { +func (l *LogicalDumper) initConfig(mysqlVersion string) error { if l.cnf == nil { return errors.New("logical dumper params is nil") } @@ -34,82 +34,6 @@ func (l *LogicalDumper) initConfig() error { return nil } -// CreateDumpCmd Create LogicalBackup Cmd -//func (l *LogicalDumper) CreateDumpCmd() (string, error) { -//var buffer bytes.Buffer -//binpath := filepath.Join(l.dbbackupHome, "/bin/mydumper") -//buffer.WriteString(binpath) -//buffer.WriteString(" -h " + l.cnf.Public.MysqlHost) -//buffer.WriteString(" -P " + l.cnf.Public.MysqlPort) -//buffer.WriteString(" -u " + l.cnf.Public.MysqlUser) -//buffer.WriteString(" -p " + l.cnf.Public.MysqlPasswd) -//buffer.WriteString(" -o " + l.cnf.Public.BackupDir + "/" + common.TargetName) -//if !l.cnf.LogicalBackup.DisableCompress { -// buffer.WriteString(" --compress") -//} -//if l.cnf.LogicalBackup.DefaultsFile != "" { -// buffer.WriteString(" --defaults-file " + l.cnf.LogicalBackup.DefaultsFile) -//} -//buffer.WriteString(" --trx-consistency-only") -//buffer.WriteString(" --long-query-retries " + cast.ToString(l.cnf.LogicalBackup.FlushRetryCount)) -//buffer.WriteString(" --set-names " + l.cnf.Public.MysqlCharset) -//buffer.WriteString(" --chunk-filesize " + cast.ToString(l.cnf.LogicalBackup.ChunkFileSize)) -//buffer.WriteString(" --long-query-retry-interval 10") -// -- buffer.WriteString(" --checksum-all") -//if l.cnf.LogicalBackup.Regex != "" { -// buffer.WriteString(fmt.Sprintf(` -x '%s'`, l.cnf.LogicalBackup.Regex)) -//} -//if common.BackupSchema && !common.BackupData { // backup schema only -// buffer.WriteString(" --no-data") -// buffer.WriteString(" --events --routines --triggers") -//} else if !common.BackupSchema && common.BackupData { // backup data only -// buffer.WriteString(" --no-schemas --no-views") -//} else if common.BackupSchema && common.BackupData { // all -// buffer.WriteString(" --events --routines --triggers") -//} -//buffer.WriteString(" --threads " + strconv.Itoa(l.cnf.LogicalBackup.Threads)) -// buffer.WriteString(" " + l.cnf.LogicalBackup.ExtraOpt) -// buffer.WriteString(" > logs/mydumper_`date +%w`.log 2>&1") -// -// cmdStr := buffer.String() -// logger.Log.Info(fmt.Sprintf("build backup cmd_line: %s", cmdStr)) -// return cmdStr, nil -//} - -//// Execute Execute logical dump command -//func (l *LogicalDumper) Execute(enableTimeOut bool) error { -// cmdStr, err := l.CreateDumpCmd() -// if err != nil { -// logger.Log.Error("Failed to create the cmd_line of dumping backup, error: ", err) -// return err -// } -// -// if enableTimeOut { -// timeDiffUnix, err := GetMaxRunningTime(l.cnf.Public.BackupTimeOut) -// if err != nil { -// return err -// } -// ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(timeDiffUnix))*time.Second) -// defer cancel() -// -// // execute command with timeout -// res, exeErr := exec.CommandContext(ctx, "/bin/bash", "-c", cmdStr).CombinedOutput() -// logger.Log.Info("execute dumping logical backup with timeout, result:", string(res)) -// if exeErr != nil { -// logger.Log.Error("Failed to execute dumping logical backup, error:", exeErr) -// return exeErr -// } -// } else { -// res, exeErr := exec.Command("/bin/bash", "-c", cmdStr).CombinedOutput() -// logger.Log.Info("execute dumping logical backup without timeout, result:", string(res)) -// if exeErr != nil { -// logger.Log.Error("Failed to execute dumping logical backup, error:", exeErr) -// return exeErr -// } -// } -// return nil -//} - // Execute excute dumping backup with logical backup tool func (l *LogicalDumper) Execute(enableTimeOut bool) error { binPath := filepath.Join(l.dbbackupHome, "/bin/mydumper") diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go index e0df64168f..713026535d 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go @@ -21,13 +21,13 @@ import ( type PhysicalDumper struct { cnf *config.BackupConfig dbbackupHome string - mysqlVersion string + mysqlVersion string // parsed + isOfficial bool innodbCmd InnodbCommand storageEngine string - isOfficial bool } -func (p *PhysicalDumper) initConfig() error { +func (p *PhysicalDumper) initConfig(mysqlVersion string) error { if p.cnf == nil { return errors.New("logical dumper params is nil") } @@ -43,11 +43,7 @@ func (p *PhysicalDumper) initConfig() error { defer func() { _ = db.Close() }() - versionStr, verErr := mysqlconn.GetMysqlVersion(db) - if verErr != nil { - return verErr - } - p.mysqlVersion, p.isOfficial = util.VersionParser(versionStr) + p.mysqlVersion, p.isOfficial = util.VersionParser(mysqlVersion) p.storageEngine, err = mysqlconn.GetStorageEngine(db) if err != nil { return err @@ -61,85 +57,6 @@ func (p *PhysicalDumper) initConfig() error { return nil } -//// CreateDumpCmd Create PhysicalBackup Cmd(Innodb) -//func (p *PhysicalDumper) CreateDumpCmd() (string, error) { -// //var buffer bytes.Buffer -// //binpath := filepath.Join(p.dbbackupHome, "/bin/xtrabackup", p.innodbCmd.innobackupexBin) -// //binpath2 := filepath.Join(p.dbbackupHome, "/bin/xtrabackup", p.innodbCmd.xtrabackupBin) -// //buffer.WriteString(binpath) -// ////buffer.WriteString(" --defaults-file=" + p.cnf.PhysicalBackup.DefaultsFile) -// //buffer.WriteString(" --host=" + p.cnf.Public.MysqlHost) -// //buffer.WriteString(" --port=" + p.cnf.Public.MysqlPort) -// //buffer.WriteString(" --user=" + p.cnf.Public.MysqlUser) -// //buffer.WriteString(" --password=" + p.cnf.Public.MysqlPasswd) -// //buffer.WriteString(" --ibbackup=" + binpath2) -// //buffer.WriteString(" --no-timestamp --compress") // --compress=qpress -// //targetPath := filepath.Join(p.cnf.Public.BackupDir, common.TargetName) -// //if strings.Compare(p.mysqlVersion, "005007000") < 0 { -// // buffer.WriteString(" " + targetPath) -// //} else { -// // buffer.WriteString(" --target-dir=" + targetPath + " --backup --binlog-info=ON") -// //} -// //if p.cnf.PhysicalBackup.Threads > 0 { -// // buffer.WriteString(" --compress-thread=" + strconv.Itoa(p.cnf.PhysicalBackup.Threads)) -// // buffer.WriteString(" --parallel=" + strconv.Itoa(p.cnf.PhysicalBackup.Threads)) -// //} -// //if p.cnf.PhysicalBackup.Throttle > 0 { -// // buffer.WriteString(" --throttle=" + strconv.Itoa(p.cnf.PhysicalBackup.Throttle)) -// //} -// //buffer.WriteString(" --lazy-backup-non-innodb --wait-last-flush=2") -// //if strings.ToLower(p.cnf.Public.MysqlRole) == cst.RoleSlave { -// // buffer.WriteString(" --slave-info --safe-slave-backup") -// //} -// buffer.WriteString(" " + p.cnf.PhysicalBackup.ExtraOpt) -// buffer.WriteString(" > logs/xtrabackup_`date +%w`.log 2>&1") -// cmdStr := buffer.String() -// logger.Log.Info(fmt.Sprintf("build backup cmd_line: %s", cmdStr)) -// return cmdStr, nil -//} - -//// Execute Execute physical dump command -//func (p *PhysicalDumper) Execute(enableTimeOut bool) error { -// var cmdStr string -// var err error -// if p.storageEngine == "innodb" { -// cmdStr, err = p.CreateDumpCmd() -// if err != nil { -// logger.Log.Error("Failed to create the cmd_line of dumping backup, error: ", err) -// return err -// } -// } else { -// logger.Log.Error(fmt.Sprintf("This is a unknown StorageEngine: %s", p.storageEngine)) -// err := fmt.Errorf("unknown StorageEngine: %s", p.storageEngine) -// return err -// } -// -// if enableTimeOut { -// timeDiffUinx, err := GetMaxRunningTime(p.cnf.Public.BackupTimeOut) -// if err != nil { -// return err -// } -// ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(timeDiffUinx))*time.Second) -// defer cancel() -// -// // execute command with timeout -// res, exeErr := exec.CommandContext(ctx, "/bin/bash", "-c", cmdStr).CombinedOutput() -// logger.Log.Info("execute dumping physical backup with timeout, result:", string(res)) -// if exeErr != nil { -// logger.Log.Error("Failed to execute dumping physical backup, error: ", exeErr) -// return exeErr -// } -// } else { -// res, exeErr := exec.Command("/bin/bash", "-c", cmdStr).CombinedOutput() -// logger.Log.Info("execute dumping physical backup without timeout, result:", string(res)) -// if exeErr != nil { -// logger.Log.Error("Failed to execute dumping physical backup, error: ", exeErr) -// return exeErr -// } -// } -// return nil -//} - // Execute excute dumping backup with physical backup tool func (p *PhysicalDumper) Execute(enableTimeOut bool) error { if p.storageEngine != "innodb" { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go index ca206fdd51..a26228c9e6 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go @@ -26,7 +26,8 @@ var ( // CmdZstd zstd command path, need call SetEnv to init CmdZstd = "" // CmdQpress qpress command path, need call SetEnv to init - CmdQpress = "" + CmdQpress = "" + XbcryptBin = "" ) // ChooseXtrabackupTool Decide the version of xtrabackup tool @@ -77,6 +78,13 @@ func (i *InnodbCommand) ChooseXtrabackupTool(mysqlVersion string, isOfficial boo return nil } +// GetXbcryptBin get xbcrypt path for encryption, maybe for mydumper to encrypt file +func GetXbcryptBin(mysqlVersion string, isOfficial bool) string { + innoCmdBin := InnodbCommand{} + _ = innoCmdBin.ChooseXtrabackupTool(mysqlVersion, isOfficial) + return innoCmdBin.xbcryptBin +} + // SetEnv set env variables func SetEnv(backupType string, mysqlVersionStr string) error { exePath, err := os.Executable() diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go index 2ea0bd1576..7dbc214814 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go @@ -1,9 +1,11 @@ package backupexe import ( - "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn" "strings" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" ) @@ -18,20 +20,21 @@ func ExecuteBackup(cnf *config.BackupConfig) error { defer func() { _ = db.Close() }() - versionStr, verErr := mysqlconn.GetMysqlVersion(db) - if verErr != nil { - return verErr + versionStr, err := mysqlconn.GetMysqlVersion(db) + if err != nil { + return err } if envErr := SetEnv(cnf.Public.BackupType, versionStr); envErr != nil { return envErr } + mysqlVersion, isOfficial := util.VersionParser(versionStr) + XbcryptBin = GetXbcryptBin(mysqlVersion, isOfficial) dumper, err := BuildDumper(cnf) if err != nil { return err } - - if err := dumper.initConfig(); err != nil { + if err := dumper.initConfig(mysqlVersion); err != nil { return err } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go index ecc88668ec..b46499621b 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go @@ -143,7 +143,6 @@ func CheckDiskSpace(backupDir string, mysqlPort int) error { if err != nil { return err } - return nil if diskSpaceInfo.Free < backupSize*2 { err = errors.New("free space is not enough") return err diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/release_package.sh b/dbm-services/mysql/db-tools/mysql-dbbackup/release_package.sh index 858e78b8ef..be93b87824 100755 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/release_package.sh +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/release_package.sh @@ -12,17 +12,9 @@ function build_package() { exit 1 fi # skip go build, we hope txsql/community has the same dbbackup binary - sh ./build.sh -s -t $1 + make package VERSION=$VERSION DIST=$1 } -echo "go build dbbackup" -rm -f build/dbbackup -make && [ -f build/dbbackup ] -if [ $? -gt 0 ];then - echo "go build dbbackup failed" - exit 1 -fi - # txsql echo echo "#################### build txsql ####################" diff --git a/dbm-services/mysql/db-tools/mysql-rotatebinlog/cmd/subcmd_version.go b/dbm-services/mysql/db-tools/mysql-rotatebinlog/cmd/subcmd_version.go new file mode 100644 index 0000000000..b05baf5a24 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-rotatebinlog/cmd/subcmd_version.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// versionCmd represents the version command +var subCmdVersion = &cobra.Command{ + Use: "version", + Short: "version information", + Long: `version information about buildStamp, gitHash`, + Run: func(cmd *cobra.Command, args []string) { + printVersion() + }, +} +var version = "" +var buildStamp = "" +var gitHash = "" + +func init() { + rootCmd.AddCommand(subCmdVersion) +} +func printVersion() { + fmt.Printf("Version: %s, GitHash: %s, BuildAt: %s\n", version, gitHash, buildStamp) +} diff --git a/dbm-ui/backend/dbm_init/medium/Dockerfile b/dbm-ui/backend/dbm_init/medium/Dockerfile index 949103fc5e..82240f0991 100644 --- a/dbm-ui/backend/dbm_init/medium/Dockerfile +++ b/dbm-ui/backend/dbm_init/medium/Dockerfile @@ -136,8 +136,6 @@ WORKDIR / RUN set -ex && \ cd /blueking-dbm/dbm-services/mysql/db-tools/dbactuator && \ make VERSION=$(/lock.sh mysql actuator version) GITHASH=$(/lock.sh mysql actuator commitId) -j4 && \ - cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-dbbackup && \ - sh build.sh -t txsql && \ cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-table-checksum && \ make release-bin VERSION=$(/lock.sh mysql mysql-checksum version) GITHASH=$(/lock.sh mysql mysql-checksum commitId) -j4 && \ cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-crond && \ @@ -147,6 +145,13 @@ RUN set -ex && \ cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-monitor && \ make release-bin VERSION=$(/lock.sh mysql mysql-monitor version) GITHASH=$(/lock.sh mysql mysql-monitor commitId) -j4 +## 编译 dbbackup-go community，下载依赖 +RUN set -ex && \ + cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-dbbackup && \ + download_url=https://github.com/TencentBlueKing/blueking-dbm/releases/download && \ + wget -qO dbbackup-go-deps-community.tar.gz ${download_url}/v1.0.0/dbbackup-go-deps-community.tar.gz && \ + make package VERSION=$(/lock.sh mysql mysql-dbbackup version) DIST=community GITHASH=$(/lock.sh mysql mysql-dbbackup commitId) -j4 + FROM medium-builder as redis-medium-builder diff --git a/dbm-ui/backend/dbm_init/medium/medium.lock b/dbm-ui/backend/dbm_init/medium/medium.lock index bbc660ac91..493a8f5f2b 100644 --- a/dbm-ui/backend/dbm_init/medium/medium.lock +++ b/dbm-ui/backend/dbm_init/medium/medium.lock @@ -29,9 +29,9 @@ mysql: name: dbactuator version: 1.0.2 - dbbackup: - buildPath: /blueking-dbm/dbm-services/mysql/db-tools/mysql-dbbackup/build/dbbackup-go-txsql.tar.gz + buildPath: /blueking-dbm/dbm-services/mysql/db-tools/mysql-dbbackup/build/dbbackup-go-community.tar.gz commitId: d451a96ded6f360315b236dddcec5a8db80e7c9f - name: dbbackup-go-txsql.tar.gz + name: dbbackup-go-community.tar.gz version: 1.0.1 - mysql-checksum: buildPath: /blueking-dbm/dbm-services/mysql/db-tools/mysql-table-checksum/build/mysql-checksum.tar.gz