fix(redis): tendisplus如果数据盘和备份盘是同一个盘,则通过硬链接方式备份 TencentBlueKing#6184

dbm-services/common/go-pubpkg/cmutil/osinfo.go

@@ -1,6 +1,7 @@
 package cmutil
 
 import (
+	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -55,7 +56,7 @@ type DiskPartInfo struct {
 	// UsedPercent 在os层面看到的磁盘利用率，包括 reserved (Used + Reserved) / Total
 	UsedPercent float64 `json:"used_percent"`
 	// UsedPercentReal Used / (Total - Reserved), stat.UsedPercent
-	//UsedPercentReal   float64 `json:"used_percent_real"`
+	// UsedPercentReal   float64 `json:"used_percent_real"`
 	InodesTotal       uint64  `json:"inodes_total"`
 	InodesUsed        uint64  `json:"inodes_used"`
 	InodesUsedPercent float64 `json:"inodes_used_percent"`
@@ -85,7 +86,7 @@ func GetDiskPartInfo(path string, checkDevice bool) (*DiskPartInfo, error) {
 			}
 		}
 		if info.Device == "" {
-			//CentOS6.2 stat --format has no %m
+			// CentOS6.2 stat --format has no %m
 			return nil, errors.Errorf("fail to get device(mounted %s) for path %s", info.Mountpoint, info.Path)
 		}
 		if info.Mountpoint != info.Path {
@@ -123,3 +124,34 @@ func GetCPUInfo() (*CPUInfo, error) {
 	}
 	return &CPUInfo{CoresLogical: cores}, nil
 }
+
+// GetTopLevelDir returns the top-level directory of a given path.
+func GetTopLevelDir(path string) (string, error) {
+	realPath, err := filepath.EvalSymlinks(path) // resolve symlinks to their real paths
+	if err != nil {
+		return "", err
+	}
+	realPath = filepath.Clean(realPath)
+	// Extract the top-level directory
+	parentDir := filepath.Dir(realPath)
+	for parentDir != "/" && parentDir != "." {
+		realPath = parentDir
+		parentDir = filepath.Dir(realPath)
+	}
+	return realPath, nil
+}
+
+// IsSameTopLevelDir 判断dir1 和  dir2是否是同一个顶级目录
+// 1. dir1 dir2 必须是存在的
+// 2. 比如 dir1 = /home/abc, dir2 = /home/abc/def,顶级目录都是 /home,所以返回true
+func IsSameTopLevelDir(dir1, dir2 string) (bool, error) {
+	topDir1, err1 := GetTopLevelDir(dir1)
+	if err1 != nil {
+		return false, err1
+	}
+	topDir2, err2 := GetTopLevelDir(dir2)
+	if err2 != nil {
+		return false, err2
+	}
+	return topDir1 == topDir2, nil
+}

dbm-services/go.work.sum

@@ -1645,6 +1645,7 @@ golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
 golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

dbm-services/redis/db-tools/dbactuator/models/myredis/client.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"dbm-services/common/go-pubpkg/cmutil"
 	"dbm-services/redis/db-tools/dbactuator/mylog"
 	"dbm-services/redis/db-tools/dbactuator/pkg/consts"
 	"dbm-services/redis/db-tools/dbactuator/pkg/util"
@@ -551,6 +552,14 @@ func (db *RedisClient) TendisplusBackup(targetDir string) (ret string, err error
 		return "", err
 	}
 	cmd := []interface{}{"backup", targetDir}
+	dataDir, _ := db.GetDir()
+	if dataDir != "" {
+		isSameTopLevelDir, _ := cmutil.IsSameTopLevelDir(dataDir, targetDir)
+		if isSameTopLevelDir {
+			// 如果备份目录 和 数据目录 在同一级目录,则添加 ckpt,代表用硬链接方式备份
+			cmd = append(cmd, "ckpt")
+		}
+	}
 	res, err := db.InstanceClient.Do(context.TODO(), cmd...).Result()
 	if err != nil {
 		err = fmt.Errorf("%+v fail,err:%v,addr:%s", cmd, err, db.Addr)

dbm-services/redis/db-tools/dbmon/models/myredis/client.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"dbm-services/common/go-pubpkg/cmutil"
 	"dbm-services/redis/db-tools/dbmon/mylog"
 	"dbm-services/redis/db-tools/dbmon/pkg/consts"
 	"dbm-services/redis/db-tools/dbmon/util"
@@ -631,6 +632,14 @@ func (db *RedisClient) TendisplusBackup(targetDir string) (ret string, err error
 		return "", err
 	}
 	cmd := []interface{}{"backup", targetDir}
+	dataDir, _ := db.GetDir()
+	if dataDir != "" {
+		isSameTopLevelDir, _ := cmutil.IsSameTopLevelDir(dataDir, targetDir)
+		if isSameTopLevelDir {
+			// 如果备份目录 和 数据目录 在同一级目录,则添加 ckpt,代表用硬链接方式备份
+			cmd = append(cmd, "ckpt")
+		}
+	}
 	res, err := db.InstanceClient.Do(context.TODO(), cmd...).Result()
 	if err != nil {
 		err = fmt.Errorf("%+v fail,err:%v,addr:%s", cmd, err, db.Addr)
@@ -660,6 +669,7 @@ func (db *RedisClient) TendisplusBackupAndWaitForDone(targetDir string) (err err
 	if err != nil {
 		return err
 	}
+
 	count := 0 // 每分钟输出一次日志
 	var msg string
 	var inProgress bool

dbm-services/redis/db-tools/dbmon/pkg/monitormemoryusage/monitormemoryusage.go

@@ -0,0 +1,46 @@
+// Package monitormemoryusage TODO
+package monitormemoryusage
+
+import (
+	"os"
+	"runtime"
+	"time"
+
+	"dbm-services/mongodb/db-tools/dbactuator/mylog"
+	"dbm-services/redis/db-tools/dbmon/config"
+	"dbm-services/redis/db-tools/dbmon/pkg/consts"
+	"dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup"
+	"dbm-services/redis/db-tools/dbmon/pkg/redisfullbackup"
+	"dbm-services/redis/db-tools/dbmon/pkg/redismaxmemory"
+	"dbm-services/redis/db-tools/dbmon/util"
+)
+
+// MonitorMemoryUsg 检测内存使用情况
+func MonitorMemoryUsg() {
+	var m runtime.MemStats
+	var times int64 = 0
+	binlogBackupGlob := redisbinlogbackup.GetGlobRedisBinlogBackupJob(config.GlobalConf)
+	fullBackupGlob := redisfullbackup.GetGlobRedisFullCheckJob(config.GlobalConf)
+	checkFullBackGlob := redisfullbackup.GetGlobRedisFullCheckJob(config.GlobalConf)
+	maxmemoryGlob := redismaxmemory.GetGlobRedisMaxmemorySet(config.GlobalConf)
+	for {
+		time.Sleep(5 * time.Second)
+		times++
+		runtime.ReadMemStats(&m)
+		// 分配内存超过500M就终止程序,等待crontab自动拉起
+		if m.Alloc >= 500*consts.MiByte {
+			if binlogBackupGlob.IsRunning || fullBackupGlob.IsRunning ||
+				checkFullBackGlob.IsRunning || maxmemoryGlob.IsRunning {
+				mylog.Logger.Error("dbmon memory usage %s exceed 500M, but binlogbackup or fullbackup or maxmemory is running",
+					util.SizeToHumanStr(int64(m.Alloc)))
+				continue
+			}
+			mylog.Logger.Error("dbmon memory usage %s exceed 500M,now exit", util.SizeToHumanStr(int64(m.Alloc)))
+			os.Exit(1)
+		}
+		// 超过100MB,每隔2分钟秒打印一次内存使用情况
+		if m.Alloc >= 100*consts.MiByte && times%24 == 0 {
+			mylog.Logger.Error("dbmon memory usage %s exceed 100M", util.SizeToHumanStr(int64(m.Alloc)))
+		}
+	}
+}