From 2831915001443fe21e48bbce042e960ba25d372b Mon Sep 17 00:00:00 2001 From: lukemakeit <2302063437@qq.com> Date: Mon, 12 Aug 2024 20:34:10 +0800 Subject: [PATCH] =?UTF-8?q?fix(redis):=20tendisplus=E5=A6=82=E6=9E=9C?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=98=E5=92=8C=E5=A4=87=E4=BB=BD=E7=9B=98?= =?UTF-8?q?=E6=98=AF=E5=90=8C=E4=B8=80=E4=B8=AA=E7=9B=98,=E5=88=99?= =?UTF-8?q?=E9=80=9A=E8=BF=87=E7=A1=AC=E9=93=BE=E6=8E=A5=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=20#6184?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/go-pubpkg/cmutil/osinfo.go | 36 ++++++++++++++- dbm-services/go.work.sum | 1 + .../dbactuator/models/myredis/client.go | 9 ++++ .../db-tools/dbmon/models/myredis/client.go | 10 ++++ .../monitormemoryusage/monitormemoryusage.go | 46 +++++++++++++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 dbm-services/redis/db-tools/dbmon/pkg/monitormemoryusage/monitormemoryusage.go diff --git a/dbm-services/common/go-pubpkg/cmutil/osinfo.go b/dbm-services/common/go-pubpkg/cmutil/osinfo.go index a2850a75ee..6469381685 100644 --- a/dbm-services/common/go-pubpkg/cmutil/osinfo.go +++ b/dbm-services/common/go-pubpkg/cmutil/osinfo.go @@ -1,6 +1,7 @@ package cmutil import ( + "path/filepath" "strings" "github.com/pkg/errors" @@ -55,7 +56,7 @@ type DiskPartInfo struct { // UsedPercent 在os层面看到的磁盘利用率，包括 reserved (Used + Reserved) / Total UsedPercent float64 `json:"used_percent"` // UsedPercentReal Used / (Total - Reserved), stat.UsedPercent - //UsedPercentReal float64 `json:"used_percent_real"` + // UsedPercentReal float64 `json:"used_percent_real"` InodesTotal uint64 `json:"inodes_total"` InodesUsed uint64 `json:"inodes_used"` InodesUsedPercent float64 `json:"inodes_used_percent"` @@ -85,7 +86,7 @@ func GetDiskPartInfo(path string, checkDevice bool) (*DiskPartInfo, error) { } } if info.Device == "" { - //CentOS6.2 stat --format has no %m + // CentOS6.2 stat --format has no %m return nil, errors.Errorf("fail to get device(mounted %s) for path %s", info.Mountpoint, info.Path) } if info.Mountpoint != info.Path { @@ -123,3 +124,34 @@ func GetCPUInfo() (*CPUInfo, error) { } return &CPUInfo{CoresLogical: cores}, nil } + +// GetTopLevelDir returns the top-level directory of a given path. +func GetTopLevelDir(path string) (string, error) { + realPath, err := filepath.EvalSymlinks(path) // resolve symlinks to their real paths + if err != nil { + return "", err + } + realPath = filepath.Clean(realPath) + // Extract the top-level directory + parentDir := filepath.Dir(realPath) + for parentDir != "/" && parentDir != "." { + realPath = parentDir + parentDir = filepath.Dir(realPath) + } + return realPath, nil +} + +// IsSameTopLevelDir 判断dir1 和 dir2是否是同一个顶级目录 +// 1. dir1 dir2 必须是存在的 +// 2. 比如 dir1 = /home/abc, dir2 = /home/abc/def,顶级目录都是 /home,所以返回true +func IsSameTopLevelDir(dir1, dir2 string) (bool, error) { + topDir1, err1 := GetTopLevelDir(dir1) + if err1 != nil { + return false, err1 + } + topDir2, err2 := GetTopLevelDir(dir2) + if err2 != nil { + return false, err2 + } + return topDir1 == topDir2, nil +} diff --git a/dbm-services/go.work.sum b/dbm-services/go.work.sum index d215f5d650..cc74cb48b0 100644 --- a/dbm-services/go.work.sum +++ b/dbm-services/go.work.sum @@ -1645,6 +1645,7 @@ golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/dbm-services/redis/db-tools/dbactuator/models/myredis/client.go b/dbm-services/redis/db-tools/dbactuator/models/myredis/client.go index 95ce961b1f..5ae81ab992 100644 --- a/dbm-services/redis/db-tools/dbactuator/models/myredis/client.go +++ b/dbm-services/redis/db-tools/dbactuator/models/myredis/client.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/redis/db-tools/dbactuator/mylog" "dbm-services/redis/db-tools/dbactuator/pkg/consts" "dbm-services/redis/db-tools/dbactuator/pkg/util" @@ -551,6 +552,14 @@ func (db *RedisClient) TendisplusBackup(targetDir string) (ret string, err error return "", err } cmd := []interface{}{"backup", targetDir} + dataDir, _ := db.GetDir() + if dataDir != "" { + isSameTopLevelDir, _ := cmutil.IsSameTopLevelDir(dataDir, targetDir) + if isSameTopLevelDir { + // 如果备份目录 和 数据目录 在同一级目录,则添加 ckpt,代表用硬链接方式备份 + cmd = append(cmd, "ckpt") + } + } res, err := db.InstanceClient.Do(context.TODO(), cmd...).Result() if err != nil { err = fmt.Errorf("%+v fail,err:%v,addr:%s", cmd, err, db.Addr) diff --git a/dbm-services/redis/db-tools/dbmon/models/myredis/client.go b/dbm-services/redis/db-tools/dbmon/models/myredis/client.go index 01647ffe1f..f73441fb56 100644 --- a/dbm-services/redis/db-tools/dbmon/models/myredis/client.go +++ b/dbm-services/redis/db-tools/dbmon/models/myredis/client.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/redis/db-tools/dbmon/mylog" "dbm-services/redis/db-tools/dbmon/pkg/consts" "dbm-services/redis/db-tools/dbmon/util" @@ -631,6 +632,14 @@ func (db *RedisClient) TendisplusBackup(targetDir string) (ret string, err error return "", err } cmd := []interface{}{"backup", targetDir} + dataDir, _ := db.GetDir() + if dataDir != "" { + isSameTopLevelDir, _ := cmutil.IsSameTopLevelDir(dataDir, targetDir) + if isSameTopLevelDir { + // 如果备份目录 和 数据目录 在同一级目录,则添加 ckpt,代表用硬链接方式备份 + cmd = append(cmd, "ckpt") + } + } res, err := db.InstanceClient.Do(context.TODO(), cmd...).Result() if err != nil { err = fmt.Errorf("%+v fail,err:%v,addr:%s", cmd, err, db.Addr) @@ -660,6 +669,7 @@ func (db *RedisClient) TendisplusBackupAndWaitForDone(targetDir string) (err err if err != nil { return err } + count := 0 // 每分钟输出一次日志 var msg string var inProgress bool diff --git a/dbm-services/redis/db-tools/dbmon/pkg/monitormemoryusage/monitormemoryusage.go b/dbm-services/redis/db-tools/dbmon/pkg/monitormemoryusage/monitormemoryusage.go new file mode 100644 index 0000000000..499030dc45 --- /dev/null +++ b/dbm-services/redis/db-tools/dbmon/pkg/monitormemoryusage/monitormemoryusage.go @@ -0,0 +1,46 @@ +// Package monitormemoryusage TODO +package monitormemoryusage + +import ( + "os" + "runtime" + "time" + + "dbm-services/mongodb/db-tools/dbactuator/mylog" + "dbm-services/redis/db-tools/dbmon/config" + "dbm-services/redis/db-tools/dbmon/pkg/consts" + "dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup" + "dbm-services/redis/db-tools/dbmon/pkg/redisfullbackup" + "dbm-services/redis/db-tools/dbmon/pkg/redismaxmemory" + "dbm-services/redis/db-tools/dbmon/util" +) + +// MonitorMemoryUsg 检测内存使用情况 +func MonitorMemoryUsg() { + var m runtime.MemStats + var times int64 = 0 + binlogBackupGlob := redisbinlogbackup.GetGlobRedisBinlogBackupJob(config.GlobalConf) + fullBackupGlob := redisfullbackup.GetGlobRedisFullCheckJob(config.GlobalConf) + checkFullBackGlob := redisfullbackup.GetGlobRedisFullCheckJob(config.GlobalConf) + maxmemoryGlob := redismaxmemory.GetGlobRedisMaxmemorySet(config.GlobalConf) + for { + time.Sleep(5 * time.Second) + times++ + runtime.ReadMemStats(&m) + // 分配内存超过500M就终止程序,等待crontab自动拉起 + if m.Alloc >= 500*consts.MiByte { + if binlogBackupGlob.IsRunning || fullBackupGlob.IsRunning || + checkFullBackGlob.IsRunning || maxmemoryGlob.IsRunning { + mylog.Logger.Error("dbmon memory usage %s exceed 500M, but binlogbackup or fullbackup or maxmemory is running", + util.SizeToHumanStr(int64(m.Alloc))) + continue + } + mylog.Logger.Error("dbmon memory usage %s exceed 500M,now exit", util.SizeToHumanStr(int64(m.Alloc))) + os.Exit(1) + } + // 超过100MB,每隔2分钟秒打印一次内存使用情况 + if m.Alloc >= 100*consts.MiByte && times%24 == 0 { + mylog.Logger.Error("dbmon memory usage %s exceed 100M", util.SizeToHumanStr(int64(m.Alloc))) + } + } +}