From aea7eec11e4192e17fe279c7eab57df72d6fc1bc Mon Sep 17 00:00:00 2001 From: seanlook Date: Mon, 13 Nov 2023 16:58:36 +0800 Subject: [PATCH] =?UTF-8?q?feat(mysql):=20=E5=A4=87=E4=BB=BD=E6=94=AF?= =?UTF-8?q?=E6=8C=81openssl/xbcrypt=E5=8A=A0=E5=AF=86=20close=20#1759?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/go-pubpkg/cmutil/cryptcmd.go | 92 ++++++++++++ .../common/go-pubpkg/cmutil/sizebytes.go | 33 +++-- .../common/go-pubpkg/iocrypt/example.go | 40 ++++++ .../common/go-pubpkg/iocrypt/filecrypt.go | 95 ++++++++++++ dbm-services/common/go-pubpkg/iocrypt/gpg.go | 1 + .../common/go-pubpkg/iocrypt/iocrypt.go | 31 ++++ .../common/go-pubpkg/iocrypt/openssl.go | 59 ++++++++ .../common/go-pubpkg/iocrypt/rsa_util.go | 135 ++++++++++++++++++ .../common/go-pubpkg/iocrypt/streamcrypt.go | 64 +++++++++ .../common/go-pubpkg/iocrypt/xbcrypt.go | 55 +++++++ .../mysql-dbbackup/cmd/subcmd_dump.go | 33 +++-- .../db-tools/mysql-dbbackup/docs/UserGuide.md | 62 ++++++++ .../mysql-dbbackup/pkg/config/logical.go | 8 +- .../mysql-dbbackup/pkg/config/public.go | 60 +++++--- .../mysql-dbbackup/pkg/src/backupexe/env.go | 40 ++++-- .../pkg/src/backupexe/indexfile.go | 45 +++--- .../pkg/src/backupexe/tarball.go | 64 +++++---- .../pkg/src/dbareport/backup_result.go | 7 +- .../pkg/src/dbareport/report.go | 18 ++- .../db-tools/mysql-dbbackup/pkg/util/misc.go | 1 + .../db-tools/mysql-dbbackup/pkg/util/tar.go | 44 ++++-- .../db-tools/mysql-dbbackup/test.2000.ini | 6 + 22 files changed, 878 insertions(+), 115 deletions(-) create mode 100644 dbm-services/common/go-pubpkg/cmutil/cryptcmd.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/example.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/filecrypt.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/gpg.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/iocrypt.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/openssl.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/rsa_util.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/streamcrypt.go create mode 100644 dbm-services/common/go-pubpkg/iocrypt/xbcrypt.go diff --git a/dbm-services/common/go-pubpkg/cmutil/cryptcmd.go b/dbm-services/common/go-pubpkg/cmutil/cryptcmd.go new file mode 100644 index 0000000000..2c2b3d4426 --- /dev/null +++ b/dbm-services/common/go-pubpkg/cmutil/cryptcmd.go @@ -0,0 +1,92 @@ +package cmutil + +import ( + "fmt" + "os/exec" + "strings" + + "github.com/pkg/errors" + + "dbm-services/common/go-pubpkg/iocrypt" +) + +type EncryptOpt struct { + // EncryptEnable 是否启用备份文件加密（对称加密），加密密码 passphrase 随机生成 + // EncryptEnable 为 true 时，EncryptTool EncryptPublicKey 有效 + EncryptEnable bool `ini:"EncryptEnable" json:"encrypt_enable" ` + // 加密工具，支持 openssl,xbcrypt，如果是xbcrypt 请指定路径 + EncryptCmd string `ini:"EncryptCmd" json:"encrypt_cmd"` + // EncryptAlgo encrypt algorithm, leave it empty has default algorithm + // openssl [aes-256-cbc, aes-128-cbc, sm4-cbc] + // xbcrypt [AES256, AES192, AES128] + EncryptAlgo iocrypt.AlgoType `ini:"EncryptElgo" json:"encrypt_algo"` + // EncryptPublicKey public key 文件，对 passphrase 加密，上报加密字符串 + // 需要对应的平台 私钥 secret key 才能对 加密后的passphrase 解密 + // EncryptPublicKey 如果为空，会上报密码，仅测试用途 + EncryptPublicKey string `ini:"EncryptPublicKey" json:"encrypt_public_key"` + + encryptTool iocrypt.EncryptTool + passPhrase string + encryptedPassPhrase string +} + +// SetEncryptTool tool can not change outside +func (e *EncryptOpt) SetEncryptTool(t iocrypt.EncryptTool) { + e.encryptTool = t +} + +// GetEncryptTool return encryptTool +// should Init first +func (e *EncryptOpt) GetEncryptTool() iocrypt.EncryptTool { + return e.encryptTool +} + +// GetEncryptedKey return encryptedPassPhrase to report +// should Init first +func (e *EncryptOpt) GetEncryptedKey() string { + return e.encryptedPassPhrase +} + +// GetPassphrase return passPhrase +// should Init first +func (e *EncryptOpt) GetPassphrase() string { + return e.passPhrase +} + +// Init 判断加密工具合法性 +// 生成公钥加密后的密码 +func (e *EncryptOpt) Init() (err error) { + if e.EncryptCmd == "" { + e.EncryptCmd = "openssl" + } + if _, err = exec.LookPath(e.EncryptCmd); err != nil { + return err + } + e.passPhrase = RandomString(32) // symmetric encrypt key to encrypt files + if e.EncryptPublicKey == "" { + e.encryptedPassPhrase = e.passPhrase // not encrypted actually, just for report + } else { + if e.encryptedPassPhrase, err = iocrypt.EncryptStringWithPubicKey(e.passPhrase, e.EncryptPublicKey); err != nil { + return err + } + } + if strings.Contains(e.EncryptCmd, "openssl") { + if e.EncryptAlgo == "" { + e.EncryptAlgo = iocrypt.AlgoAES256CBC + } + e.encryptTool = iocrypt.Openssl{CryptCmd: e.EncryptCmd, EncryptElgo: e.EncryptAlgo, EncryptKey: e.passPhrase} + } else if strings.Contains(e.EncryptCmd, "xbcrypt") { + if e.EncryptAlgo == "" { + e.EncryptAlgo = iocrypt.AlgoAES256 + } + e.encryptTool = iocrypt.Xbcrypt{CryptCmd: e.EncryptCmd, EncryptElgo: e.EncryptAlgo, EncryptKey: e.passPhrase} + } else { + return errors.Errorf("unknown EncryptTool command: %s", e.EncryptCmd) + } + return nil +} + +func (e *EncryptOpt) String() string { + return fmt.Sprintf("EncryptOpt{Enable:%t, Cmd:%s, Algo:%s PublicKeyFile:%s encryptedKey:%s}", + e.EncryptEnable, e.EncryptCmd, e.EncryptAlgo, e.EncryptPublicKey, e.encryptedPassPhrase) +} diff --git a/dbm-services/common/go-pubpkg/cmutil/sizebytes.go b/dbm-services/common/go-pubpkg/cmutil/sizebytes.go index 31d7d7be0a..6ec247c976 100644 --- a/dbm-services/common/go-pubpkg/cmutil/sizebytes.go +++ b/dbm-services/common/go-pubpkg/cmutil/sizebytes.go @@ -20,9 +20,10 @@ func ViperGetSizeInBytesE(key string) (int64, error) { } // ParseSizeInBytesE converts strings like 1GB or 12 mb into an unsigned integer number of bytes -// withB indicate where sizeStr has suffix b/B -func ParseSizeInBytesE(sizeStr string) (int64, error) { - sizeStr = strings.TrimSpace(sizeStr) +// if sizeStr has no suffix b/B, append to it +// b will be treated as B, can not handle 1GiB like +func ParseSizeInBytesE(sizeStr string) (size int64, err error) { + sizeStr = strings.TrimSpace(strings.ToLower(sizeStr)) if unicode.ToLower(rune(sizeStr[len(sizeStr)-1])) != 'b' { sizeStr += "b" } @@ -51,13 +52,23 @@ func ParseSizeInBytesE(sizeStr string) (int64, error) { } } } - size, err := cast.ToInt64E(sizeStr) - if err != nil { - return -1, errors.Errorf("parse failed to bytes: %s", sizeStr) - } else if size < 0 { + if strings.Contains(sizeStr, ".") { + sizeFloat, err := cast.ToFloat64E(sizeStr) + if err != nil { + return -1, errors.Errorf("parse failed to bytes: %s", sizeStr) + } + size = safeMulFloat(sizeFloat, int64(multiplier)) + } else { + size, err = cast.ToInt64E(sizeStr) + if err != nil { + return -1, errors.Errorf("parse failed to bytes: %s", sizeStr) + } + size = safeMul(size, int64(multiplier)) + } + if size < 0 { return -2, errors.Errorf("bytes canot be negative: %s", sizeStr) } - return safeMul(size, int64(multiplier)), nil + return size, nil } func safeMul(a, b int64) int64 { @@ -68,6 +79,12 @@ func safeMul(a, b int64) int64 { return c } +// safeMulFloat WARN: bytes will lose precision +func safeMulFloat(a float64, b int64) int64 { + c := a * float64(b) + return int64(c) +} + // ParseSizeInBytes 将 gb, MB 转换成 bytes 数字. b 不区分大小写，代表 1字节 // ignore error func ParseSizeInBytes(sizeStr string) int64 { diff --git a/dbm-services/common/go-pubpkg/iocrypt/example.go b/dbm-services/common/go-pubpkg/iocrypt/example.go new file mode 100644 index 0000000000..fe6fe123ec --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/example.go @@ -0,0 +1,40 @@ +package iocrypt + +import ( + "fmt" + "io" + "os" +) + +func doEncryptFile() error { + srcFilename := "aaa.tar" + srcFile, err := os.Open(srcFilename) + if err != nil { + return err + } + defer srcFile.Close() + + encryptTool := Openssl{CryptCmd: "openssl", EncryptElgo: AlgoAES256CBC, EncryptKey: "aaa"} + encryptFile, err := os.OpenFile(srcFilename+"."+encryptTool.DefaultSuffix(), + os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer encryptFile.Close() + + xbw, err := FileEncryptWriter(&encryptTool, encryptFile) + if err != nil { + return err + } + written, err := io.Copy(xbw, srcFile) + err1 := xbw.Close() + if err != nil { + fmt.Println("write eeeee") + return err + } + if err1 != nil { + return err1 + } + fmt.Println("written bytes", written) + return nil +} diff --git a/dbm-services/common/go-pubpkg/iocrypt/filecrypt.go b/dbm-services/common/go-pubpkg/iocrypt/filecrypt.go new file mode 100644 index 0000000000..d1161602e6 --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/filecrypt.go @@ -0,0 +1,95 @@ +package iocrypt + +import ( + "bytes" + "context" + "fmt" + "io" + "os/exec" + "time" + + "github.com/pkg/errors" +) + +// FileEncrypter file encrypter +type FileEncrypter struct { + CryptTool EncryptTool + CryptTimeout time.Duration + stdin io.WriteCloser + stderr bytes.Buffer + cmd *exec.Cmd +} + +// InitWriter 包装 encrypt writer +// 会启动加密命令，等待标准输入。当 InitWriter 执行成功，写入结束后需要调用 Close 关闭 stdin +func (r *FileEncrypter) InitWriter(w io.Writer) error { + var err error + if r.cmd == nil { + //if exec.LookPath(r.CryptTool.Name()) + r.cmd, err = r.CryptTool.BuildCommand(context.Background()) + if err != nil { + return err + } + r.stdin, err = r.cmd.StdinPipe() + if err != nil { + return err + } + r.cmd.Stdout = w + r.cmd.Stderr = &r.stderr + //r.cmd.WaitDelay = 100 * time.Millisecond + + if err := r.cmd.Start(); err != nil { + return err + } + time.Sleep(100 * time.Millisecond) + go r.cmd.Wait() + if r.cmd.ProcessState != nil && !r.cmd.ProcessState.Success() { + return errors.Errorf("fail to start encrypt tool: %s", r.stderr.String()) + } + // if ProcessState == nil means the process is still running + if r.CryptTimeout == 0 { + r.CryptTimeout = 999 * time.Hour + } + } + return errors.WithStack(err) +} + +// String for print +func (r *FileEncrypter) String() string { + // may need to remove sensitive key + return fmt.Sprintf("FileEncrypter{Cmd:%s, ExecTimeout:%s Suffix:%s}", + r.cmd.String(), r.CryptTimeout, r.CryptTool.DefaultSuffix()) +} + +// Write implement io.Write +func (r *FileEncrypter) Write(p []byte) (int, error) { + var err error + if r.stdin == nil { + return 0, errors.New("encrypt has no stdin to read from") + } + written, err := r.stdin.Write(p) + //time.Sleep(0.5 * time.Second) + return written, errors.WithStack(err) +} + +// Close 关闭进程，会检查是否有错误输出 +// 用户需要自己关闭外层的 file reader 和 writer, InitWriter 成功了才需要调用 Close +func (r *FileEncrypter) Close() error { + _ = r.stdin.Close() + if r.cmd.ProcessState == nil { + return nil + } + if !r.cmd.ProcessState.Exited() { + if err := r.cmd.Process.Kill(); err != nil { + time.Sleep(100 * time.Millisecond) + if !r.cmd.ProcessState.Exited() { + return errors.Errorf("fail to clean encrypt process pid=%d", r.cmd.ProcessState.Pid()) + } + } else { + return nil + } + } else if r.cmd.ProcessState.ExitCode() > 0 { + return errors.Errorf("encrypt tool exited with error: %s", r.stderr.String()) + } + return nil +} diff --git a/dbm-services/common/go-pubpkg/iocrypt/gpg.go b/dbm-services/common/go-pubpkg/iocrypt/gpg.go new file mode 100644 index 0000000000..b56053accd --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/gpg.go @@ -0,0 +1 @@ +package iocrypt diff --git a/dbm-services/common/go-pubpkg/iocrypt/iocrypt.go b/dbm-services/common/go-pubpkg/iocrypt/iocrypt.go new file mode 100644 index 0000000000..cc42782bed --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/iocrypt.go @@ -0,0 +1,31 @@ +package iocrypt + +import ( + "context" + "io" + "os/exec" + + "github.com/pkg/errors" +) + +// EncryptTool usually Symmetric encryption +type EncryptTool interface { + BuildCommand(ctx context.Context) (*exec.Cmd, error) + DefaultSuffix() string + Name() string +} + +// AlgoType algorithm type +type AlgoType string + +// FileEncryptWriter new +func FileEncryptWriter(cryptTool EncryptTool, w io.Writer) (io.WriteCloser, error) { + if cryptTool == nil { + return nil, errors.New("no crypt tool provide") + } + xbw := &FileEncrypter{CryptTool: cryptTool} + if err := xbw.InitWriter(w); err != nil { + return nil, err + } + return xbw, nil +} diff --git a/dbm-services/common/go-pubpkg/iocrypt/openssl.go b/dbm-services/common/go-pubpkg/iocrypt/openssl.go new file mode 100644 index 0000000000..68981b2889 --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/openssl.go @@ -0,0 +1,59 @@ +package iocrypt + +import ( + "context" + "fmt" + "os/exec" + + "github.com/pkg/errors" + "golang.org/x/exp/slices" +) + +// Openssl EncryptTool +type Openssl struct { + CryptCmd string + EncryptElgo AlgoType + // EncryptKey passphrase used to encrypt something + EncryptKey string + // EncryptKeyFile passphrase from file used to encrypt something + EncryptKeyFile string +} + +const ( + AlgoAES256CBC AlgoType = "aes-256-cbc" + AlgoAES192CBC AlgoType = "aes-128-cbc" + AlgoSM4CBC AlgoType = "sm4-cbc" // need openssl>=1.1.1 +) + +var opensslAllowedAlgos = []AlgoType{AlgoAES256CBC, AlgoAES192CBC, AlgoSM4CBC} + +// BuildCommand implement BuildCommand +func (e Openssl) BuildCommand(ctx context.Context) (*exec.Cmd, error) { + if !slices.Contains(opensslAllowedAlgos, e.EncryptElgo) { + return nil, errors.Errorf("unknown crypt algorithm: %s", e.EncryptElgo) + } + cmdArgs := []string{"enc", fmt.Sprintf("-%s", e.EncryptElgo), "-e", "-salt"} // >=1.1.1 "-pbkdf2" + + if e.EncryptKeyFile != "" { + cmdArgs = append(cmdArgs, "-kfile", e.EncryptKeyFile) + } else if e.EncryptKey != "" { + //keyHash := fmt.Sprintf("%x", md5.Sum([]byte(e.EncryptKey))) + if len(e.EncryptKey)%16 != 0 { + return nil, errors.Errorf("key len error(need N*16): %s", e.EncryptKey) + } + cmdArgs = append(cmdArgs, "-k", e.EncryptKey) + } else { + return nil, errors.New("no key provide") + } + return exec.CommandContext(ctx, e.CryptCmd, cmdArgs...), nil +} + +// DefaultSuffix return default suffix for encrypt tool +func (e Openssl) DefaultSuffix() string { + return "enc" +} + +// Name return encrypt tool name +func (e Openssl) Name() string { + return e.CryptCmd +} diff --git a/dbm-services/common/go-pubpkg/iocrypt/rsa_util.go b/dbm-services/common/go-pubpkg/iocrypt/rsa_util.go new file mode 100644 index 0000000000..c0077d3911 --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/rsa_util.go @@ -0,0 +1,135 @@ +package iocrypt + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "log" + "os" + + "github.com/pkg/errors" +) + +// https://gist.github.com/miguelmota/3ea9286bd1d3c2a985b67cac4ba2130a + +// GenerateKeyPair generates a new key pair +func GenerateKeyPair(bits int) (*rsa.PrivateKey, *rsa.PublicKey, error) { + privkey, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return nil, nil, err + } + return privkey, &privkey.PublicKey, nil +} + +// PrivateKeyToBytes private key to bytes +func PrivateKeyToBytes(priv *rsa.PrivateKey) []byte { + privBytes := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }, + ) + + return privBytes +} + +// PublicKeyToBytes public key to bytes +func PublicKeyToBytes(pub *rsa.PublicKey) ([]byte, error) { + pubASN1, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return nil, err + } + + pubBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PUBLIC KEY", + Bytes: pubASN1, + }) + + return pubBytes, nil +} + +// BytesToPrivateKey bytes to private key +func BytesToPrivateKey(priv []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(priv) + enc := x509.IsEncryptedPEMBlock(block) + b := block.Bytes + var err error + if enc { + log.Println("is encrypted pem block") + b, err = x509.DecryptPEMBlock(block, nil) + if err != nil { + return nil, err + } + } + key, err := x509.ParsePKCS1PrivateKey(b) + if err != nil { + return nil, err + } + return key, nil +} + +// BytesToPublicKey bytes to public key +func BytesToPublicKey(pub []byte) (*rsa.PublicKey, error) { + block, _ := pem.Decode(pub) + enc := x509.IsEncryptedPEMBlock(block) + b := block.Bytes + var err error + if enc { + log.Println("is encrypted pem block") + b, err = x509.DecryptPEMBlock(block, nil) + if err != nil { + return nil, err + } + } + ifc, err := x509.ParsePKIXPublicKey(b) + if err != nil { + return nil, err + } + key, ok := ifc.(*rsa.PublicKey) + if !ok { + return nil, errors.New("not ok") + } + return key, nil +} + +// EncryptWithPublicKey encrypts data with public key +func EncryptWithPublicKey(msg []byte, pub *rsa.PublicKey) ([]byte, error) { + //ciphertext, err := rsa.EncryptOAEP(sha256.New(), rand.Reader, pub, msg, nil) + ciphertext, err := rsa.EncryptPKCS1v15(rand.Reader, pub, msg) + if err != nil { + return nil, err + } + return ciphertext, nil +} + +// DecryptWithPrivateKey decrypts data with private key +func DecryptWithPrivateKey(ciphertext []byte, priv *rsa.PrivateKey) ([]byte, error) { + //plaintext, err := rsa.DecryptOAEP(sha256.New(), rand.Reader, priv, ciphertext, nil) + plaintext, err := rsa.DecryptPKCS1v15(rand.Reader, priv, ciphertext) + if err != nil { + return nil, err + } + return plaintext, nil +} + +// EncryptStringWithPubicKey 使用RSA公钥，加密 对称密码 +// 返回base64 +func EncryptStringWithPubicKey(passPhrase string, publicKeyFile string) (string, error) { + var cipherText string + if bs, err := os.ReadFile(publicKeyFile); err != nil { + return "", errors.Wrap(err, "fail to read encrypt public key file") + } else { + pubKey, err := BytesToPublicKey(bs) + if err != nil { + return "", errors.Wrapf(err, "fail to parse public key file %s", publicKeyFile) + } + encryptedPass, err := EncryptWithPublicKey([]byte(passPhrase), pubKey) + if err != nil { + return "", errors.Wrapf(err, "fail to encrypt key") + } + cipherText = base64.StdEncoding.EncodeToString(encryptedPass) + } + return cipherText, nil +} diff --git a/dbm-services/common/go-pubpkg/iocrypt/streamcrypt.go b/dbm-services/common/go-pubpkg/iocrypt/streamcrypt.go new file mode 100644 index 0000000000..2eaf837799 --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/streamcrypt.go @@ -0,0 +1,64 @@ +package iocrypt + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rand" + "errors" + "io" +) + +// https://gist.github.com/ayubmalik/2c973c2a7ae7e0d22ece7f5c4dfbd726 + +// EncryptedWriter wraps w with an OFB cipher stream. +func EncryptedWriter(key string, w io.Writer) (*cipher.StreamWriter, error) { + + // generate random initial value + iv := make([]byte, aes.BlockSize) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return nil, err + } + + // write clear IV to allow for decryption + n, err := w.Write(iv) + if err != nil || n != len(iv) { + return nil, errors.New("could not write initial value") + } + + block, err := newBlock(key) + if err != nil { + return nil, err + } + + stream := cipher.NewOFB(block, iv) + return &cipher.StreamWriter{S: stream, W: w}, nil +} + +// EncryptedReader wraps r with an OFB cipher stream. +func EncryptedReader(key string, r io.Reader) (*cipher.StreamReader, error) { + + // read initial value + iv := make([]byte, aes.BlockSize) + n, err := r.Read(iv) + if err != nil || n != len(iv) { + return nil, errors.New("could not read initial value") + } + + block, err := newBlock(key) + if err != nil { + return nil, err + } + + stream := cipher.NewOFB(block, iv) + return &cipher.StreamReader{S: stream, R: r}, nil +} + +func newBlock(key string) (cipher.Block, error) { + hash := md5.Sum([]byte(key)) + block, err := aes.NewCipher(hash[:]) + if err != nil { + return nil, err + } + return block, nil +} diff --git a/dbm-services/common/go-pubpkg/iocrypt/xbcrypt.go b/dbm-services/common/go-pubpkg/iocrypt/xbcrypt.go new file mode 100644 index 0000000000..f61700c83d --- /dev/null +++ b/dbm-services/common/go-pubpkg/iocrypt/xbcrypt.go @@ -0,0 +1,55 @@ +package iocrypt + +import ( + "context" + "os/exec" + + "github.com/pkg/errors" + "golang.org/x/exp/slices" +) + +// Xbcrypt EncryptTool +type Xbcrypt struct { + CryptCmd string + EncryptElgo AlgoType + EncryptKey string + EncryptKeyFile string +} + +const ( + AlgoAES256 AlgoType = "AES256" + AlgoAES192 AlgoType = "AES192" + AlgoAES128 AlgoType = "ASE128" +) + +var xbcryptAllowedAlgos = []AlgoType{AlgoAES256, AlgoAES192, AlgoAES128} + +// BuildCommand implement BuildCommand +func (e Xbcrypt) BuildCommand(ctx context.Context) (*exec.Cmd, error) { + if !slices.Contains(xbcryptAllowedAlgos, e.EncryptElgo) { + return nil, errors.Errorf("unknown crypt algorithm: %s", e.EncryptElgo) + } + cmdArgs := []string{"-a", string(e.EncryptElgo)} // –encrypt-threads + if e.EncryptKeyFile != "" { + cmdArgs = append(cmdArgs, "-f", e.EncryptKeyFile) + } else if e.EncryptKey != "" { + //keyHash := fmt.Sprintf("%x", md5.Sum([]byte(e.EncryptKey))) + if len(e.EncryptKey)%16 != 0 { + return nil, errors.Errorf("key len error(need N*16): %s", e.EncryptKey) + } + cmdArgs = append(cmdArgs, "-k", e.EncryptKey) + } else { + return nil, errors.New("no key provide") + } + return exec.CommandContext(ctx, e.CryptCmd, cmdArgs...), nil +} + +// DefaultSuffix return suffix +func (e Xbcrypt) DefaultSuffix() string { + return "xb" +} + +// Name return name +func (e Xbcrypt) Name() string { + return e.CryptCmd +} diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go index 56c2c34d70..c0aafa1d65 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/viper" + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/validate" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe" @@ -80,16 +81,25 @@ var dumpCmd = &cobra.Command{ }, } -func backupData(cnf *config.BackupConfig) error { - logger.Log.Info("begin backup") - +func backupData(cnf *config.BackupConfig) (err error) { + logger.Log.Info("Dbbackup begin") // validate dumpBackup - if err := validate.GoValidateStruct(cnf.Public, false, false); err != nil { + if err = validate.GoValidateStruct(cnf.Public, false, false); err != nil { return err } - //if err := cnf.Public.ParseDataSchemaGrant(); err != nil { - // return err - //} + if cnf.Public.EncryptOpt == nil { + cnf.Public.EncryptOpt = &cmutil.EncryptOpt{EncryptEnable: false} + } + encOpt := cnf.Public.EncryptOpt + if encOpt.EncryptEnable { + if encOpt.EncryptCmd == "xbcrypt" { + encOpt.EncryptCmd = filepath.Join(backupexe.ExecuteHome, "bin/xbcrypt") + } + if err := encOpt.Init(); err != nil { + return errors.Wrap(err, "fail to init crypt tool") + } + cnf.Public.EncryptOpt = encOpt + } cnfPublic := cnf.Public DBAReporter, err := dbareport.NewReporter(cnf) @@ -102,14 +112,7 @@ func backupData(cnf *config.BackupConfig) error { logger.Log.Error("report begin failed: ", err) return err } - logger.Log.Info("parse config file: end") - //// produce a unique targetname - //var tnameErr error - //common.TargetName, tnameErr = backupexe.GetTargetName(&cnfPublic) - //if tnameErr != nil { - // return tnameErr - //} // backup grant info if cnf.Public.IfBackupGrant() { @@ -152,7 +155,7 @@ func backupData(cnf *config.BackupConfig) error { } var baseBackupResult dbareport.BackupResult - if err := baseBackupResult.BuildBaseBackupResult(cnf, DBAReporter.Uuid); err != nil { + if err := baseBackupResult.BuildBaseBackupResult(cnf, DBAReporter); err != nil { return err } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/UserGuide.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/UserGuide.md index 4decfa3405..99ebcc61a7 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/UserGuide.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/UserGuide.md @@ -189,6 +189,67 @@ index文件内容格式为： {"backup_id":"23d29c7a-7773-11ed-b724-525400b22106","bill_id":"","status":"Success","report_time":"2022-12-09 11:39:53"} ``` +## 3.4 备份加密 +### 加密选项 +``` +[Public.EncryptOpt] +EncryptEnable = true +EncryptCmd = openssl +EncryptPublicKey = +EncryptElgo = +``` +1. EncryptEnable: 是否启用备份文件加密 + 对称加密,加密密码 passphrase 随机生成 +2. EncryptCmd: 加密工具，支持 `openssl`,`xbcrypt` + - 留空默认为 openssl + - 如果是 xbcrypt,默认从工具目录下找 `bin/xbcrypt`，也可以指定工具全路径 +3. EncryptAlgo: 加密算法，留空会有默认加密算法 + - openssl [aes-256-cbc, aes-128-cbc, sm4-cbc]，文件后缀 `.enc`。sm4-cbc 为国密对称加密算法 + - xbcrypt [AES256, AES192, AES128]，文件后缀 `.xb` +4. EncryptPublicKey: public key 文件 + - 用于 对 passphrase 加密，上报加密字符串。需要对应的平台 私钥 secret key 才能对 加密后的passphrase 解密 + - EncryptPublicKey 如果为空，会上报密码，仅测试用途 + +### EncryptPublicKey 生成示例 +``` +# 生成秘钥 +openssl genrsa -out rsa.pem 2048 +# 从秘钥文件 rsa.pem 中提取公钥 +openssl rsa -pubout -in rsa.pem -out pubkey.pem +``` +把 pubkey.pem 路径设置到 EncryptPublicKey + +### 手动解密文件 +如果没有设置 EncryptPublicKey ，可直接使用上报记录里的 key 解密，但这不安全，仅测试使用。 +如果设置了 EncryptPublicKey，先要通过私钥解密出 passphrase： +``` +// 1. 被加密密码 base64 解码成文件 +echo -n "GiySD...bbw==" |base64 -d > encrypted.key + +// 2. 使用私钥 rsa.pem 解密出 passphrase +openssl rsautl -decrypt -inkey rsa.pem -in encrypted.key + +// 3. 使用密码 passphrase 解密文件 +``` +一般 passphrase 需要从平台的页面获取，因为私钥不能泄露给使用者。 + +- openssl 解密文件 +``` +openssl aes-256-cbc -d -k your_passphrase -in backupfile.tar.enc -out backupfile.tar +``` +- xbcrypt 解密文件 +``` +xbcrypt -d -a AES256 -k your_passphrase -i backupfile.tar.xb -o backupfile.tar +``` + +### dbbackup filecrypt 解密文件 +自动识别后缀，使用对应的解密工具 +``` +dbbackup filecrypt -d -k your_passphrase \ +--remove-files -i backupfile.tar.xb -o backupfile.tar +// --source-dir /xxx/ --target-dir=/yyy +``` + # 4. 配置文件示例 ``` @@ -214,6 +275,7 @@ IOLimitMBPerSec = 500 ResultReportPath = /home/mysql/dbareport/mysql/dbbackup/result StatusReportPath = /home/mysql/dbareport/mysql/dbbackup/status + [BackupClient] Enable = false RemoteFileSystem = hdfs diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/logical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/logical.go index 846a331244..44581a2885 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/logical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/logical.go @@ -11,9 +11,11 @@ package config // LogicalBackup the config of logical backup // data or schema is controlled by Public.DataSchemaGrant type LogicalBackup struct { - ChunkFilesize uint64 `ini:"ChunkFilesize"` // split tables into chunks of this output file size. This value is in MB - Regex string `ini:"Regex"` - Threads int `ini:"Threads"` + // ChunkFilesize split tables into chunks of this output file size. This value is in MB + ChunkFilesize uint64 `ini:"ChunkFilesize"` + Regex string `ini:"Regex"` + Threads int `ini:"Threads"` + // DisableCompress disable zstd compress. compress is enable by default DisableCompress bool `ini:"DisableCompress"` FlushRetryCount int `ini:"FlushRetryCount"` DefaultsFile string `ini:"DefaultsFile"` diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go index ee71c19a7a..3177ba0513 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go @@ -15,35 +15,57 @@ import ( "golang.org/x/exp/slices" + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" ) +// Public 公共配置 type Public struct { - BkBizId int `ini:"BkBizId" validate:"required"` - BkCloudId int `ini:"BkCloudId"` - BillId string `ini:"BillId"` - BackupId string `ini:"BackupId"` - ClusterId int `ini:"ClusterId"` + // BkBizId bk_biz_id + BkBizId int `ini:"BkBizId" validate:"required"` + // BkCloudId 云区域id + BkCloudId int `ini:"BkCloudId"` + // BillId 备份单据id，例行备份为空，单据发起的备份请设置单据id + BillId string `ini:"BillId"` + // BackupId backup uuid，代表一次备份 + BackupId string `ini:"BackupId"` + // ClusterId cluster_id + ClusterId int `ini:"ClusterId"` + // ClusterAddress cluster_domain ClusterAddress string `ini:"ClusterAddress"` - ShardValue int `ini:"ShardValue"` // 分片 id，仅 spider 有用 - MysqlHost string `ini:"MysqlHost" validate:"required,ip"` - MysqlPort int `ini:"MysqlPort" validate:"required"` - MysqlUser string `ini:"MysqlUser" validate:"required"` - MysqlPasswd string `ini:"MysqlPasswd"` + // ShardValue for spider + ShardValue int `ini:"ShardValue"` // 分片 id，仅 spider 有用 + // MysqlHost backup host + MysqlHost string `ini:"MysqlHost" validate:"required,ip"` + // MysqlPort backup port + MysqlPort int `ini:"MysqlPort" validate:"required"` + // MysqlUser backup user to login + MysqlUser string `ini:"MysqlUser" validate:"required"` + // MysqlPasswd backup user's password + MysqlPasswd string `ini:"MysqlPasswd"` // DataSchemaGrant data,grant,schema,priv,all，写了 data 则只备data，不备份 schema - DataSchemaGrant string `ini:"DataSchemaGrant" validate:"required"` - BackupDir string `ini:"BackupDir" validate:"required"` - MysqlRole string `ini:"MysqlRole" validate:"required"` // oneof=master slave - MysqlCharset string `ini:"MysqlCharset"` - BackupTimeOut string `ini:"BackupTimeout"` // 备份时间阈值，格式 09:00:01 - BackupType string `ini:"BackupType" validate:"required"` // oneof=logical physical - OldFileLeftDay int `ini:"OldFileLeftDay"` // will remove old backup files before the days - TarSizeThreshold uint64 `ini:"TarSizeThreshold" validate:"required,gte=128"` // tar file size. MB - IOLimitMBPerSec int `ini:"IOLimitMBPerSec"` // tar speed, mb/s. 0 means no limit + DataSchemaGrant string `ini:"DataSchemaGrant" validate:"required"` + // BackupDir backup files to save + BackupDir string `ini:"BackupDir" validate:"required"` + MysqlRole string `ini:"MysqlRole" validate:"required"` // oneof=master slave + MysqlCharset string `ini:"MysqlCharset"` + // BackupTimeOut 备份时间阈值，格式 09:00:01 + BackupTimeOut string `ini:"BackupTimeout"` + // BackupType backup type, oneof=logical physical + BackupType string `ini:"BackupType" validate:"required"` + // OldFileLeftDay will remove old backup files before the days + OldFileLeftDay int `ini:"OldFileLeftDay"` + // TarSizeThreshold tar file size. MB + TarSizeThreshold uint64 `ini:"TarSizeThreshold" validate:"required,gte=128"` + // IOLimitMBPerSec tar speed, mb/s. 0 means no limit + IOLimitMBPerSec int `ini:"IOLimitMBPerSec"` ResultReportPath string `ini:"ResultReportPath" validate:"required"` StatusReportPath string `ini:"StatusReportPath" validate:"required"` + // EncryptOpt backup files encrypt options + EncryptOpt *cmutil.EncryptOpt `ini:"EncryptOpt"` + cnfFilename string targetName string } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go index 5a379893a3..ca206fdd51 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/env.go @@ -17,8 +17,18 @@ import ( type InnodbCommand struct { innobackupexBin string xtrabackupBin string + xbcryptBin string } +var ( + // ExecuteHome executable home dir, like /home/mysql/dbbackup-go/ + ExecuteHome = "" + // CmdZstd zstd command path, need call SetEnv to init + CmdZstd = "" + // CmdQpress qpress command path, need call SetEnv to init + CmdQpress = "" +) + // ChooseXtrabackupTool Decide the version of xtrabackup tool func (i *InnodbCommand) ChooseXtrabackupTool(mysqlVersion string, isOfficial bool) error { if !isOfficial { @@ -27,20 +37,24 @@ func (i *InnodbCommand) ChooseXtrabackupTool(mysqlVersion string, isOfficial boo // tmysql 5.5 i.innobackupexBin = "/bin/xtrabackup/innobackupex_55.pl" i.xtrabackupBin = "/bin/xtrabackup/xtrabackup_55" + i.xbcryptBin = "/bin/xtrabackup/xbcrypt" } else if strings.Compare(mysqlVersion, "005006000") >= 0 && strings.Compare(mysqlVersion, "005007000") < 0 { // tmysql 5.6 i.innobackupexBin = "/bin/xtrabackup/innobackupex_56.pl" i.xtrabackupBin = "/bin/xtrabackup/xtrabackup_56" + i.xbcryptBin = "/bin/xtrabackup/xbcrypt" } else if strings.Compare(mysqlVersion, "005007000") >= 0 && strings.Compare(mysqlVersion, "008000000") < 0 { // tmysql 5.7 i.innobackupexBin = "/bin/xtrabackup/xtrabackup_57" i.xtrabackupBin = "/bin/xtrabackup/xtrabackup_57" + i.xbcryptBin = "/bin/xtrabackup/xbcrypt_57" } else if strings.Compare(mysqlVersion, "008000000") >= 0 { // tmysql 8.0 i.innobackupexBin = "/bin/xtrabackup/xtrabackup_80" i.xtrabackupBin = "/bin/xtrabackup/xtrabackup_80" + i.xbcryptBin = "/bin/xtrabackup/xbcrypt_80" } else { return fmt.Errorf("unrecognizable mysql version") } @@ -50,10 +64,12 @@ func (i *InnodbCommand) ChooseXtrabackupTool(mysqlVersion string, isOfficial boo // official_mysql_5.7 i.innobackupexBin = "/bin/xtrabackup_official/xtrabackup_57/xtrabackup" i.xtrabackupBin = "/bin/xtrabackup_official/xtrabackup_57/xtrabackup" + i.xbcryptBin = "/bin/xtrabackup_official/xtrabackup_57/xbcrypt" } else if strings.Compare(mysqlVersion, "008000000") >= 0 { //official_mysql_8.0 i.innobackupexBin = "/bin/xtrabackup_official/xtrabackup_80/xtrabackup" i.xtrabackupBin = "/bin/xtrabackup_official/xtrabackup_80/xtrabackup" + i.xbcryptBin = "/bin/xtrabackup_official/xtrabackup_80/xbcrypt" } else { return fmt.Errorf("unrecognizable mysql version") } @@ -67,31 +83,33 @@ func SetEnv(backupType string, mysqlVersionStr string) error { if err != nil { return err } - exePath = filepath.Dir(exePath) + ExecuteHome = filepath.Dir(exePath) var libPath []string var binPath []string if strings.ToLower(backupType) == "logical" { - libPath = append(libPath, filepath.Join(exePath, "lib/libmydumper")) + libPath = append(libPath, filepath.Join(ExecuteHome, "lib/libmydumper")) } else if strings.ToLower(backupType) == "physical" { _, isOfficial := util.VersionParser(mysqlVersionStr) if !isOfficial { - libPath = append(libPath, filepath.Join(exePath, "lib/libxtra")) - libPath = append(libPath, filepath.Join(exePath, "lib/libxtra_80")) + libPath = append(libPath, filepath.Join(ExecuteHome, "lib/libxtra")) + libPath = append(libPath, filepath.Join(ExecuteHome, "lib/libxtra_80")) - binPath = append(binPath, filepath.Join(exePath, "bin/xtrabackup")) + binPath = append(binPath, filepath.Join(ExecuteHome, "bin/xtrabackup")) } else { - libPath = append(libPath, filepath.Join(exePath, "lib/libxtra_57_official/private")) - libPath = append(libPath, filepath.Join(exePath, "lib/libxtra_57_official/plugin")) - libPath = append(libPath, filepath.Join(exePath, "lib/libxtra_80_official/private")) - libPath = append(libPath, filepath.Join(exePath, "lib/libxtra_80_official/plugin")) + libPath = append(libPath, filepath.Join(ExecuteHome, "lib/libxtra_57_official/private")) + libPath = append(libPath, filepath.Join(ExecuteHome, "lib/libxtra_57_official/plugin")) + libPath = append(libPath, filepath.Join(ExecuteHome, "lib/libxtra_80_official/private")) + libPath = append(libPath, filepath.Join(ExecuteHome, "lib/libxtra_80_official/plugin")) - binPath = append(binPath, filepath.Join(exePath, "bin/xtrabackup_official")) + binPath = append(binPath, filepath.Join(ExecuteHome, "bin/xtrabackup_official")) } } else { return fmt.Errorf("setEnv: unknown backupType") } // xtrabackup --decompress 需要找到 qpress 命令 - binPath = append(binPath, filepath.Join(exePath, "bin")) + binPath = append(binPath, filepath.Join(ExecuteHome, "bin")) + CmdZstd = filepath.Join(ExecuteHome, "bin/zstd") + CmdQpress = filepath.Join(ExecuteHome, "bin/qpress") logger.Log.Info(fmt.Sprintf("export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:%s", strings.Join(libPath, ":"))) logger.Log.Info(fmt.Sprintf("export PATH=$PATH:%s", strings.Join(binPath, ":"))) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/indexfile.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/indexfile.go index 1bd471ae13..f67bc8a2c6 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/indexfile.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/indexfile.go @@ -59,26 +59,28 @@ const ( // IndexContent the content of the index file type IndexContent struct { - BackupType string `json:"backup_type"` - StorageEngine string `json:"storage_engine"` - MysqlVersion string `json:"mysql_version"` - BkBizId int `json:"bk_biz_id"` - BackupId string `json:"backup_id"` - BillId string `json:"bill_id"` - ClusterId int `json:"cluster_id"` - ClusterAddress string `json:"cluster_address"` - ShardValue int `json:"shard_value"` - BackupHost string `json:"backup_host"` - BackupPort int `json:"backup_port"` - BackupCharset string `json:"backup_charset"` - MysqlRole string `json:"mysql_role"` - DataSchemaGrant string `json:"data_schema_grant"` - ConsistentBackupTime string `json:"consistent_backup_time"` - BackupBeginTime string `json:"backup_begin_time"` - BackupEndTime string `json:"backup_end_time"` - TotalFilesize uint64 `json:"total_filesize"` - TotalFilesizeUncompress uint64 `json:"total_filesize_uncompress"` - BinlogInfo dbareport.BinlogStatusInfo `json:"binlog_info"` + BackupType string `json:"backup_type"` + StorageEngine string `json:"storage_engine"` + MysqlVersion string `json:"mysql_version"` + BkBizId int `json:"bk_biz_id"` + BackupId string `json:"backup_id"` + BillId string `json:"bill_id"` + ClusterId int `json:"cluster_id"` + ClusterAddress string `json:"cluster_address"` + ShardValue int `json:"shard_value"` + BackupHost string `json:"backup_host"` + BackupPort int `json:"backup_port"` + BackupCharset string `json:"backup_charset"` + MysqlRole string `json:"mysql_role"` + DataSchemaGrant string `json:"data_schema_grant"` + ConsistentBackupTime string `json:"consistent_backup_time"` + BackupBeginTime string `json:"backup_begin_time"` + BackupEndTime string `json:"backup_end_time"` + TotalFilesize uint64 `json:"total_filesize"` + // TotalSizeKBUncompress 压缩前大小，如果是zstd压缩会提供压缩前大小，-1,0 都是无效值。这不是精确大小，可能存在四舍五入 + TotalSizeKBUncompress int64 `json:"total_size_kb_uncompress"` + BinlogInfo dbareport.BinlogStatusInfo `json:"binlog_info"` + EncryptEnable bool `json:"encrypt_enable"` FileList []FileIndex `json:"file_list"` @@ -128,6 +130,9 @@ func (i *IndexContent) Init(cnf *config.Public, resultInfo *dbareport.BackupResu i.BackupEndTime = resultInfo.BackupEndTime i.BackupId = resultInfo.BackupId i.BinlogInfo = resultInfo.BinlogInfo + if resultInfo.EncryptedKey != "" { + i.EncryptEnable = true + } i.reData = regexp.MustCompile(ReData) i.reSchema = regexp.MustCompile(ReSchema) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/tarball.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/tarball.go index 04f0b2e0e2..514f044084 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/tarball.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/tarball.go @@ -49,15 +49,22 @@ func (p *PackageFile) MappingPackage() error { return err } - tarFileNum := 0 - dstTarName := fmt.Sprintf(`%s_%d.tar`, p.dstDir, tarFileNum) var tarSize uint64 = 0 + tarFileNum := 0 var tarUtil = util.TarWriter{IOLimitMB: p.cnf.Public.IOLimitMBPerSec} + var dstTarName = fmt.Sprintf(`%s_%d.tar`, p.dstDir, tarFileNum) + if p.cnf.Public.EncryptOpt.EncryptEnable { + logger.Log.Infof("tar file encrypt enabled for port: %d", p.cnf.Public.MysqlPort) + tarUtil.Encrypt = true + tarUtil.EncryptTool = p.cnf.Public.EncryptOpt.GetEncryptTool() + dstTarName = fmt.Sprintf(`%s_%d.tar.%s`, p.dstDir, tarFileNum, tarUtil.EncryptTool.DefaultSuffix()) + } + if err := tarUtil.New(dstTarName); err != nil { return err } defer func() { - _ = tarUtil.Close() + _ = tarUtil.Close() // the last tar file to close }() var totalSizeUncompress int64 = 0 // -1 means does not calculate size before compress @@ -78,31 +85,34 @@ func (p *PackageFile) MappingPackage() error { } else if !isFile { return nil } - p.indexFile.addFileContent(dstTarName, filename, written) - if err = os.Remove(filename); err != nil { //TODO 限速？ - logger.Log.Error("failed to remove file while taring, err:", err) - } + if totalSizeUncompress > -1 && strings.HasSuffix(filename, cst.ZstdSuffix) { - if sizeUncompress, err := readUncompressSizeForZstd("", filename); err != nil { + if sizeUncompress, err := readUncompressSizeForZstd(CmdZstd, filename); err != nil { logger.Log.Warnf("fail to readUncompressSizeForZstd for file %s, err: %s", filename, err.Error()) totalSizeUncompress = -1 } else { totalSizeUncompress += sizeUncompress } } + if err = os.Remove(filename); err != nil { //TODO 限速？ + logger.Log.Error("failed to remove file while taring, err:", err) + } tarSize += uint64(written) if tarSize >= tarSizeMaxBytes { logger.Log.Infof("need to tar file, accumulated tar size: %d bytes, dstFile: %s", tarSize, dstTarName) - if err = tarUtil.Close(); err != nil { - return err - } p.indexFile.TotalFilesize += tarSize tarSize = 0 tarFileNum++ + if err = tarUtil.Close(); err != nil { + return err + } // new tarUtil object will be used for next loop dstTarName = fmt.Sprintf(`%s_%d.tar`, p.dstDir, tarFileNum) + if p.cnf.Public.EncryptOpt.EncryptEnable { + dstTarName = fmt.Sprintf(`%s_%d.tar.%s`, p.dstDir, tarFileNum, tarUtil.EncryptTool.DefaultSuffix()) + } if err = tarUtil.New(dstTarName); err != nil { return err } @@ -111,7 +121,7 @@ func (p *PackageFile) MappingPackage() error { }) logger.Log.Infof("need to tar file, accumulated tar size: %d bytes, dstFile: %s", tarSize, dstTarName) p.indexFile.TotalFilesize += tarSize - p.indexFile.TotalFilesizeUncompress = uint64(totalSizeUncompress) + p.indexFile.TotalSizeKBUncompress = totalSizeUncompress / 1024 if walkErr != nil { logger.Log.Error("walk dir, err: ", walkErr) return walkErr @@ -188,18 +198,20 @@ func (p *PackageFile) tarballDir() error { } else if !isFile { return nil } - // TODO limit io rate when removing - if err = os.Remove(filename); err != nil { - logger.Log.Error("failed to remove file while taring, err:", err) - } if totalSizeUncompress > -1 && strings.HasSuffix(filename, cst.ZstdSuffix) { - if sizeUncompress, err := readUncompressSizeForZstd("", filename); err != nil { + if sizeUncompress, err := readUncompressSizeForZstd(CmdZstd, filename); err != nil { logger.Log.Warnf("fail to readUncompressSizeForZstd for file %s, err: %s", filename, err.Error()) totalSizeUncompress = -1 } else { totalSizeUncompress += sizeUncompress } } + + // TODO limit io rate when removing + if err = os.Remove(filename); err != nil { + logger.Log.Error("failed to remove file while taring, err:", err) + } + return nil }) if walkErr != nil { @@ -318,13 +330,8 @@ func readUncompressSizeForZstd(zstdCmd string, fileName string) (int64, error) { } } if i == 1 { - cols := strings.FieldsFunc(line, func(r rune) bool { - if r == '\t' { - return true - } - return false - }) - readableBytes := strings.ReplaceAll(strings.ReplaceAll(cols[3], "i", ""), " ", "") + cols := strings.Fields(line) + readableBytes := strings.ReplaceAll(strings.ReplaceAll(cols[4]+cols[5], "i", ""), " ", "") bytesNum, err := cmutil.ParseSizeInBytesE(readableBytes) if err != nil { return 0, errors.Wrapf(err, "fail to parse size %s for %s", readableBytes, fileName) @@ -334,3 +341,12 @@ func readUncompressSizeForZstd(zstdCmd string, fileName string) (int64, error) { } return 0, errors.Errorf("unknown error, zst -l %s output error", fileName) } + +func tarBallWithEncrypt(tarFilename string, srcFilename string) error { + encryptCmd := []string{"openssl", "enc", "-aes-256-cbc", "-salt", "-k", "aaaa", "-out", "aaaa.tar.enc"} + //encryptCmd := []string{"xbcrypt", "--encrypt=AES256"} + tarCmd := []string{"tar", "-rf", "-", "dir1"} + cmdStr := fmt.Sprintf("%s| pv | %s ", strings.Join(tarCmd, " "), strings.Join(encryptCmd, " ")) + fmt.Println(cmdStr) + return nil +} diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/backup_result.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/backup_result.go index d0a221bdb4..b155cc3297 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/backup_result.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/backup_result.go @@ -41,6 +41,8 @@ type BackupResult struct { FileSize int64 `json:"file_size"` FileType string `json:"file_type"` TaskId string `json:"task_id"` + // EncryptedKey encrypted passphrase using public key + EncryptedKey string `json:"encrypted_key"` } // BinlogStatusInfo master status and slave status @@ -140,8 +142,9 @@ func (b *BackupResult) PrepareXtraBackupInfo(cnf *config.BackupConfig) error { } // BuildBaseBackupResult Build based BackupResult -func (b *BackupResult) BuildBaseBackupResult(cnf *config.BackupConfig, uuid string) error { - b.BackupId = uuid +func (b *BackupResult) BuildBaseBackupResult(cnf *config.BackupConfig, reporter *Reporter) error { + b.BackupId = reporter.Uuid + b.EncryptedKey = reporter.EncryptedKey b.BillId = cnf.Public.BillId b.BkBizId = cnf.Public.BkBizId b.BkCloudId = cnf.Public.BkCloudId diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report.go index d95124b7ee..23fd35b9b7 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report.go @@ -4,7 +4,6 @@ package dbareport import ( "bytes" "encoding/json" - "errors" "fmt" "os" "os/exec" @@ -12,6 +11,8 @@ import ( "strings" "time" + "github.com/pkg/errors" + "dbm-services/common/go-pubpkg/backupclient" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" @@ -21,8 +22,11 @@ import ( // Reporter TODO type Reporter struct { - cfg *config.BackupConfig + cfg *config.BackupConfig + // Uuid backup_uuid 代表一次备份 Uuid string + // EncryptedKey 如果备份加密，上报加密 key。加密短语 key 会通过 rsa 加密成密文 再上报 + EncryptedKey string } // NewReporter TODO @@ -38,7 +42,15 @@ func NewReporter(cfg *config.BackupConfig) (reporter *Reporter, err error) { return nil, err } } - + if cfg.Public.EncryptOpt.EncryptEnable { + if ekey := cfg.Public.EncryptOpt.GetEncryptedKey(); len(ekey) <= 32 { + logger.Log.Warnf("Not safe because EncryptPublicKey is not set, key=%s", ekey) + reporter.EncryptedKey = ekey + } else { + logger.Log.Infof("Passphrase encrypted=%s passphrase=%s", ekey, cfg.Public.EncryptOpt.GetPassphrase()) + reporter.EncryptedKey = ekey + } + } return reporter, nil } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go index b46499621b..ecc88668ec 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go @@ -143,6 +143,7 @@ func CheckDiskSpace(backupDir string, mysqlPort int) error { if err != nil { return err } + return nil if diskSpaceInfo.Free < backupSize*2 { err = errors.New("free space is not enough") return err diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/tar.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/tar.go index 89c3f7f3e4..e16a169d77 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/tar.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/tar.go @@ -3,10 +3,13 @@ package util import ( "archive/tar" + "fmt" + "io" "os" "sync" "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/iocrypt" ) // TarWriterFiles tarFile: writer object @@ -17,21 +20,37 @@ type TarWriterFiles struct { // TarWriter TODO type TarWriter struct { - IOLimitMB int + IOLimitMB int + EncryptTool iocrypt.EncryptTool + Encrypt bool - tarSize uint64 - destFileWriter *os.File - tarWriter *tar.Writer - mu sync.Mutex + tarSize uint64 + destFileWriter *os.File + destEncryptWriter io.WriteCloser + tarWriter *tar.Writer + mu sync.Mutex } // New TODO +// init tarWriter destFileWriter destEncryptWriter +// will open destFile +// destFileWriter or destEncryptWriter need to close outside +// need to call tarWriter.Close() func (t *TarWriter) New(dstTarName string) (err error) { t.destFileWriter, err = os.Create(dstTarName) if err != nil { return err } - t.tarWriter = tar.NewWriter(t.destFileWriter) + if t.Encrypt { + t.destEncryptWriter, err = iocrypt.FileEncryptWriter(t.EncryptTool, t.destFileWriter) + if err != nil { + fmt.Println("TarWriter new error", err) + return err + } + t.tarWriter = tar.NewWriter(t.destEncryptWriter) + } else { + t.tarWriter = tar.NewWriter(t.destFileWriter) + } return nil } @@ -53,6 +72,7 @@ func (t *TarWriter) WriteTar(header *tar.Header, srcFile string) (isFile bool, w return isFile, 0, err } defer rFile.Close() + written, err = cmutil.IOLimitRate(t.tarWriter, rFile, int64(t.IOLimitMB)) if err != nil { return isFile, 0, err @@ -60,15 +80,19 @@ func (t *TarWriter) WriteTar(header *tar.Header, srcFile string) (isFile bool, w return isFile, written, nil } -// Close TODO +// Close tar writer +// will close destFile +// close won't reset IOLimitMB EncryptTool, could reuse it with new tarFilename func (t *TarWriter) Close() error { if err := t.tarWriter.Close(); err != nil { + _ = t.destFileWriter.Close() return err } - if err := t.destFileWriter.Close(); err != nil { - return err + if t.Encrypt { + return t.destEncryptWriter.Close() + } else { + return t.destFileWriter.Close() } - return nil } /*func tarCmd(filepath string, cnf *parsecnf.CnfShared) error { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini b/dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini index 790a8d9138..b4b48064f9 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini @@ -22,6 +22,11 @@ IOLimitMBPerSec = 500 ResultReportPath= /data/git-code/dbbackup StatusReportPath= /data/git-code/dbbackup +[Public.EncryptOpt] +EncryptEnable = false +EncryptCmd = openssl +EncryptPublicKey = +EncryptElgo = [BackupClient] Enable = false @@ -29,6 +34,7 @@ RemoteFileSystem = cos FileTag = MYSQL_FULL_BACKUP DoChecksum = true BackupClientBin = /usr/local/backup_client/bin/backup_client +CosInfoFile = /home/mysql/.cosinfo.toml [LogicalBackup] ChunkFilesize = 2048