feat(mysql): 备份支持openssl/xbcrypt加密 close #1759

dbm-services/common/go-pubpkg/cmutil/cryptcmd.go

@@ -0,0 +1,92 @@
+package cmutil
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"dbm-services/common/go-pubpkg/iocrypt"
+)
+
+type EncryptOpt struct {
+	// EncryptEnable 是否启用备份文件加密（对称加密），加密密码 passphrase 随机生成
+	// EncryptEnable 为 true 时，EncryptTool EncryptPublicKey 有效
+	EncryptEnable bool `ini:"EncryptEnable" json:"encrypt_enable" `
+	// 加密工具，支持 openssl,xbcrypt，如果是xbcrypt 请指定路径
+	EncryptCmd string `ini:"EncryptCmd" json:"encrypt_cmd"`
+	// EncryptAlgo encrypt algorithm, leave it empty has default algorithm
+	//  openssl [aes-256-cbc, aes-128-cbc, sm4-cbc]
+	//  xbcrypt [AES256, AES192, AES128]
+	EncryptAlgo iocrypt.AlgoType `ini:"EncryptElgo" json:"encrypt_algo"`
+	// EncryptPublicKey public key 文件，对 passphrase 加密，上报加密字符串
+	// 需要对应的平台 私钥 secret key 才能对 加密后的passphrase 解密
+	// EncryptPublicKey 如果为空，会上报密码，仅测试用途
+	EncryptPublicKey string `ini:"EncryptPublicKey" json:"encrypt_public_key"`
+
+	encryptTool         iocrypt.EncryptTool
+	passPhrase          string
+	encryptedPassPhrase string
+}
+
+// SetEncryptTool tool can not change outside
+func (e *EncryptOpt) SetEncryptTool(t iocrypt.EncryptTool) {
+	e.encryptTool = t
+}
+
+// GetEncryptTool return encryptTool
+// should Init first
+func (e *EncryptOpt) GetEncryptTool() iocrypt.EncryptTool {
+	return e.encryptTool
+}
+
+// GetEncryptedKey return encryptedPassPhrase to report
+// should Init first
+func (e *EncryptOpt) GetEncryptedKey() string {
+	return e.encryptedPassPhrase
+}
+
+// GetPassphrase return passPhrase
+// should Init first
+func (e *EncryptOpt) GetPassphrase() string {
+	return e.passPhrase
+}
+
+// Init 判断加密工具合法性
+// 生成公钥加密后的密码
+func (e *EncryptOpt) Init() (err error) {
+	if e.EncryptCmd == "" {
+		e.EncryptCmd = "openssl"
+	}
+	if _, err = exec.LookPath(e.EncryptCmd); err != nil {
+		return err
+	}
+	e.passPhrase = RandomString(32) // symmetric encrypt key to encrypt files
+	if e.EncryptPublicKey == "" {
+		e.encryptedPassPhrase = e.passPhrase // not encrypted actually, just for report
+	} else {
+		if e.encryptedPassPhrase, err = iocrypt.EncryptStringWithPubicKey(e.passPhrase, e.EncryptPublicKey); err != nil {
+			return err
+		}
+	}
+	if strings.Contains(e.EncryptCmd, "openssl") {
+		if e.EncryptAlgo == "" {
+			e.EncryptAlgo = iocrypt.AlgoAES256CBC
+		}
+		e.encryptTool = iocrypt.Openssl{CryptCmd: e.EncryptCmd, EncryptElgo: e.EncryptAlgo, EncryptKey: e.passPhrase}
+	} else if strings.Contains(e.EncryptCmd, "xbcrypt") {
+		if e.EncryptAlgo == "" {
+			e.EncryptAlgo = iocrypt.AlgoAES256
+		}
+		e.encryptTool = iocrypt.Xbcrypt{CryptCmd: e.EncryptCmd, EncryptElgo: e.EncryptAlgo, EncryptKey: e.passPhrase}
+	} else {
+		return errors.Errorf("unknown EncryptTool command: %s", e.EncryptCmd)
+	}
+	return nil
+}
+
+func (e *EncryptOpt) String() string {
+	return fmt.Sprintf("EncryptOpt{Enable:%t, Cmd:%s, Algo:%s PublicKeyFile:%s encryptedKey:%s}",
+		e.EncryptEnable, e.EncryptCmd, e.EncryptAlgo, e.EncryptPublicKey, e.encryptedPassPhrase)
+}

dbm-services/common/go-pubpkg/cmutil/sizebytes.go

@@ -20,9 +20,10 @@ func ViperGetSizeInBytesE(key string) (int64, error) {
 }
 
 // ParseSizeInBytesE converts strings like 1GB or 12 mb into an unsigned integer number of bytes
-// withB indicate where sizeStr has suffix b/B
-func ParseSizeInBytesE(sizeStr string) (int64, error) {
-	sizeStr = strings.TrimSpace(sizeStr)
+// if sizeStr has no suffix b/B, append to it
+// b will be treated as B, can not handle 1GiB like
+func ParseSizeInBytesE(sizeStr string) (size int64, err error) {
+	sizeStr = strings.TrimSpace(strings.ToLower(sizeStr))
 	if unicode.ToLower(rune(sizeStr[len(sizeStr)-1])) != 'b' {
 		sizeStr += "b"
 	}
@@ -51,13 +52,23 @@ func ParseSizeInBytesE(sizeStr string) (int64, error) {
 			}
 		}
 	}
-	size, err := cast.ToInt64E(sizeStr)
-	if err != nil {
-		return -1, errors.Errorf("parse failed to bytes: %s", sizeStr)
-	} else if size < 0 {
+	if strings.Contains(sizeStr, ".") {
+		sizeFloat, err := cast.ToFloat64E(sizeStr)
+		if err != nil {
+			return -1, errors.Errorf("parse failed to bytes: %s", sizeStr)
+		}
+		size = safeMulFloat(sizeFloat, int64(multiplier))
+	} else {
+		size, err = cast.ToInt64E(sizeStr)
+		if err != nil {
+			return -1, errors.Errorf("parse failed to bytes: %s", sizeStr)
+		}
+		size = safeMul(size, int64(multiplier))
+	}
+	if size < 0 {
 		return -2, errors.Errorf("bytes canot be negative: %s", sizeStr)
 	}
-	return safeMul(size, int64(multiplier)), nil
+	return size, nil
 }
 
 func safeMul(a, b int64) int64 {
@@ -68,6 +79,12 @@ func safeMul(a, b int64) int64 {
 	return c
 }
 
+// safeMulFloat WARN: bytes will lose precision
+func safeMulFloat(a float64, b int64) int64 {
+	c := a * float64(b)
+	return int64(c)
+}
+
 // ParseSizeInBytes 将 gb, MB 转换成 bytes 数字. b 不区分大小写，代表 1字节
 // ignore error
 func ParseSizeInBytes(sizeStr string) int64 {

dbm-services/common/go-pubpkg/iocrypt/example.go

@@ -0,0 +1,40 @@
+package iocrypt
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+func doEncryptFile() error {
+	srcFilename := "aaa.tar"
+	srcFile, err := os.Open(srcFilename)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	encryptTool := Openssl{CryptCmd: "openssl", EncryptElgo: AlgoAES256CBC, EncryptKey: "aaa"}
+	encryptFile, err := os.OpenFile(srcFilename+"."+encryptTool.DefaultSuffix(),
+		os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	defer encryptFile.Close()
+
+	xbw, err := FileEncryptWriter(&encryptTool, encryptFile)
+	if err != nil {
+		return err
+	}
+	written, err := io.Copy(xbw, srcFile)
+	err1 := xbw.Close()
+	if err != nil {
+		fmt.Println("write eeeee")
+		return err
+	}
+	if err1 != nil {
+		return err1
+	}
+	fmt.Println("written bytes", written)
+	return nil
+}

dbm-services/common/go-pubpkg/iocrypt/filecrypt.go

@@ -0,0 +1,95 @@
+package iocrypt
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// FileEncrypter file encrypter
+type FileEncrypter struct {
+	CryptTool    EncryptTool
+	CryptTimeout time.Duration
+	stdin        io.WriteCloser
+	stderr       bytes.Buffer
+	cmd          *exec.Cmd
+}
+
+// InitWriter 包装 encrypt writer
+// 会启动加密命令，等待标准输入。当 InitWriter 执行成功，写入结束后需要调用 Close 关闭 stdin
+func (r *FileEncrypter) InitWriter(w io.Writer) error {
+	var err error
+	if r.cmd == nil {
+		//if exec.LookPath(r.CryptTool.Name())
+		r.cmd, err = r.CryptTool.BuildCommand(context.Background())
+		if err != nil {
+			return err
+		}
+		r.stdin, err = r.cmd.StdinPipe()
+		if err != nil {
+			return err
+		}
+		r.cmd.Stdout = w
+		r.cmd.Stderr = &r.stderr
+		//r.cmd.WaitDelay = 100 * time.Millisecond
+
+		if err := r.cmd.Start(); err != nil {
+			return err
+		}
+		time.Sleep(100 * time.Millisecond)
+		go r.cmd.Wait()
+		if r.cmd.ProcessState != nil && !r.cmd.ProcessState.Success() {
+			return errors.Errorf("fail to start encrypt tool: %s", r.stderr.String())
+		}
+		// if ProcessState == nil means the process is still running
+		if r.CryptTimeout == 0 {
+			r.CryptTimeout = 999 * time.Hour
+		}
+	}
+	return errors.WithStack(err)
+}
+
+// String for print
+func (r *FileEncrypter) String() string {
+	// may need to remove sensitive key
+	return fmt.Sprintf("FileEncrypter{Cmd:%s, ExecTimeout:%s Suffix:%s}",
+		r.cmd.String(), r.CryptTimeout, r.CryptTool.DefaultSuffix())
+}
+
+// Write implement io.Write
+func (r *FileEncrypter) Write(p []byte) (int, error) {
+	var err error
+	if r.stdin == nil {
+		return 0, errors.New("encrypt has no stdin to read from")
+	}
+	written, err := r.stdin.Write(p)
+	//time.Sleep(0.5 * time.Second)
+	return written, errors.WithStack(err)
+}
+
+// Close 关闭进程，会检查是否有错误输出
+// 用户需要自己关闭外层的 file reader 和 writer, InitWriter 成功了才需要调用 Close
+func (r *FileEncrypter) Close() error {
+	_ = r.stdin.Close()
+	if r.cmd.ProcessState == nil {
+		return nil
+	}
+	if !r.cmd.ProcessState.Exited() {
+		if err := r.cmd.Process.Kill(); err != nil {
+			time.Sleep(100 * time.Millisecond)
+			if !r.cmd.ProcessState.Exited() {
+				return errors.Errorf("fail to clean encrypt process pid=%d", r.cmd.ProcessState.Pid())
+			}
+		} else {
+			return nil
+		}
+	} else if r.cmd.ProcessState.ExitCode() > 0 {
+		return errors.Errorf("encrypt tool exited with error: %s", r.stderr.String())
+	}
+	return nil
+}

dbm-services/common/go-pubpkg/iocrypt/gpg.go

@@ -0,0 +1 @@
+package iocrypt

dbm-services/common/go-pubpkg/iocrypt/iocrypt.go

@@ -0,0 +1,31 @@
+package iocrypt
+
+import (
+	"context"
+	"io"
+	"os/exec"
+
+	"github.com/pkg/errors"
+)
+
+// EncryptTool usually Symmetric encryption
+type EncryptTool interface {
+	BuildCommand(ctx context.Context) (*exec.Cmd, error)
+	DefaultSuffix() string
+	Name() string
+}
+
+// AlgoType algorithm type
+type AlgoType string
+
+// FileEncryptWriter new
+func FileEncryptWriter(cryptTool EncryptTool, w io.Writer) (io.WriteCloser, error) {
+	if cryptTool == nil {
+		return nil, errors.New("no crypt tool provide")
+	}
+	xbw := &FileEncrypter{CryptTool: cryptTool}
+	if err := xbw.InitWriter(w); err != nil {
+		return nil, err
+	}
+	return xbw, nil
+}

dbm-services/common/go-pubpkg/iocrypt/openssl.go

@@ -0,0 +1,59 @@
+package iocrypt
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+
+	"github.com/pkg/errors"
+	"golang.org/x/exp/slices"
+)
+
+// Openssl EncryptTool
+type Openssl struct {
+	CryptCmd    string
+	EncryptElgo AlgoType
+	// EncryptKey passphrase used to encrypt something
+	EncryptKey string
+	// EncryptKeyFile passphrase from file used to encrypt something
+	EncryptKeyFile string
+}
+
+const (
+	AlgoAES256CBC AlgoType = "aes-256-cbc"
+	AlgoAES192CBC AlgoType = "aes-128-cbc"
+	AlgoSM4CBC    AlgoType = "sm4-cbc" // need openssl>=1.1.1
+)
+
+var opensslAllowedAlgos = []AlgoType{AlgoAES256CBC, AlgoAES192CBC, AlgoSM4CBC}
+
+// BuildCommand implement BuildCommand
+func (e Openssl) BuildCommand(ctx context.Context) (*exec.Cmd, error) {
+	if !slices.Contains(opensslAllowedAlgos, e.EncryptElgo) {
+		return nil, errors.Errorf("unknown crypt algorithm: %s", e.EncryptElgo)
+	}
+	cmdArgs := []string{"enc", fmt.Sprintf("-%s", e.EncryptElgo), "-e", "-salt"} // >=1.1.1 "-pbkdf2"
+
+	if e.EncryptKeyFile != "" {
+		cmdArgs = append(cmdArgs, "-kfile", e.EncryptKeyFile)
+	} else if e.EncryptKey != "" {
+		//keyHash := fmt.Sprintf("%x", md5.Sum([]byte(e.EncryptKey)))
+		if len(e.EncryptKey)%16 != 0 {
+			return nil, errors.Errorf("key len error(need N*16): %s", e.EncryptKey)
+		}
+		cmdArgs = append(cmdArgs, "-k", e.EncryptKey)
+	} else {
+		return nil, errors.New("no key provide")
+	}
+	return exec.CommandContext(ctx, e.CryptCmd, cmdArgs...), nil
+}
+
+// DefaultSuffix return default suffix for encrypt tool
+func (e Openssl) DefaultSuffix() string {
+	return "enc"
+}
+
+// Name return encrypt tool name
+func (e Openssl) Name() string {
+	return e.CryptCmd
+}