Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(redis): redis 备份上报 添加 segment 、slots 等信息,同时添加 cache_backup_mode(aof/rdb) #1028

Merged
merged 1 commit into from
Sep 12, 2023
Merged

fix(redis): redis 备份上报 添加 segment 、slots 等信息,同时添加 cache_backup_mode(aof/rdb) #1028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions dbm-services/redis/db-tools/dbactuator/example/bkdbmon_install.example.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,17 +59,24 @@ bk-dbmon安装:
"bk_cloud_id":"246",
"app":"testapp",
"app_name":"测试app",
"cluster_domain":"tendisx.aaaa.testapp.db",
"cluster_domain":"cache.aaaa.testapp.db",
"cluster_name":"aaaa",
"cluster_type":"PredixyTendisplusCluster",
"cluster_type":"TwemproxyRedisInstance",
"meta_role":"redis_master",
"server_ip":"127.0.0.1",
"server_ports":[
30000,
30001,
30002,
30003
]
],
"server_shards":{
"a.a.a.a:12000":"0-104999",
"a.a.a.a:12001":"105000-209999",
"a.a.a.a:12002":"210000-314999",
"a.a.a.a:12003":"315000-419999"
},
"cache_backup_mode":"rdb"
},
{
"bk_biz_id":"200500194",
Expand All @@ -86,7 +93,9 @@ bk-dbmon安装:
31001,
31002,
31003
]
],
"server_shards":{},
"cache_backup_mode":""
}
]
}
Expand Down
34 changes: 32 additions & 2 deletions dbm-services/redis/db-tools/dbactuator/models/myredis/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,8 +712,8 @@ func (db *RedisClient) Sscan(keyname string, cursor uint64, match string, count
fields, retCursor, err = db.InstanceClient.SScan(context.TODO(), keyname, cursor, match, count).Result()
}
if err != nil && err != redis.Nil {
mylog.Logger.Error("Redis 'sscan %s %d match %s count %s' command fail,err:%v,addr:%s", keyname, cursor, match, count,
err, db.Addr)
mylog.Logger.Error("Redis 'sscan %s %d match %s count %d' command fail,err:%v,addr:%s",
keyname, cursor, match, count, err, db.Addr)
return fields, 0, err
}
return fields, retCursor, nil
Expand Down Expand Up @@ -1761,3 +1761,33 @@ func (db *RedisClient) GetClusterNodesStr() (ret string, err error) {
}
return
}

// RedisClusterGetMasterNode 获取master节点信息(如果 addr是master则返回它的node信息,否则找到它的masterID,进而找到master的node信息)
func (db *RedisClient) RedisClusterGetMasterNode(addr string) (masterNode *ClusterNodeData, err error) {
addrToNodes, err := db.GetAddrMapToNodes()
if err != nil {
return
}
myNode, ok := addrToNodes[addr]
if !ok {
err = fmt.Errorf("addr:%s not found in cluster nodes", addr)
mylog.Logger.Error(err.Error())
return
}
if myNode.GetRole() == consts.RedisMasterRole {
masterNode = myNode
return
}
masterNodeID := myNode.MasterID
idToNode, err := db.GetNodeIDMapToNodes()
if err != nil {
return
}
masterNode, ok = idToNode[masterNodeID]
if !ok {
err = fmt.Errorf("masterNodeID:%s not found in cluster nodes", masterNodeID)
mylog.Logger.Error(err.Error())
return
}
return
}
26 changes: 14 additions & 12 deletions dbm-services/redis/db-tools/dbactuator/pkg/atomjobs/atomredis/bkdbmon_install.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,19 @@ import (

// ConfServerItem servers配置项
type ConfServerItem struct {
BkBizID string `json:"bk_biz_id" yaml:"bk_biz_id" validate:"required"`
BkCloudID int64 `json:"bk_cloud_id" yaml:"bk_cloud_id"`
App string `json:"app" yaml:"app" validate:"required"`
AppName string `json:"app_name" yaml:"app_name" validate:"required"`
ClusterDomain string `json:"cluster_domain" yaml:"cluster_domain" validate:"required"`
ClusterName string `json:"cluster_name" yaml:"cluster_name" validate:"required"`
ClusterType string `json:"cluster_type" yaml:"cluster_type" validate:"required"`
MetaRole string `json:"meta_role" yaml:"meta_role" validate:"required"`
ServerIP string `json:"server_ip" yaml:"server_ip" validate:"required"`
ServerPorts []int `json:"server_ports" yaml:"server_ports" validate:"required"`
Shard string `json:"shard" yaml:"shard"`
BkBizID string `json:"bk_biz_id" yaml:"bk_biz_id" validate:"required"`
BkCloudID int64 `json:"bk_cloud_id" yaml:"bk_cloud_id"`
App string `json:"app" yaml:"app" validate:"required"`
AppName string `json:"app_name" yaml:"app_name" validate:"required"`
ClusterDomain string `json:"cluster_domain" yaml:"cluster_domain" validate:"required"`
ClusterName string `json:"cluster_name" yaml:"cluster_name" validate:"required"`
ClusterType string `json:"cluster_type" yaml:"cluster_type" validate:"required"`
MetaRole string `json:"meta_role" yaml:"meta_role" validate:"required"`
ServerIP string `json:"server_ip" yaml:"server_ip" validate:"required"`
ServerPorts []int `json:"server_ports" yaml:"server_ports" validate:"required"`
ServerShards map[string]string `json:"server_shards" yaml:"server_shards"`
CacheBackupMode string `json:"cache_backup_mode" yaml:"cache_backup_mode"` // aof or rdb
Shard string `json:"shard" yaml:"shard"`
}

// BkDbmonInstallParams 安装参数
Expand Down Expand Up @@ -90,7 +92,7 @@ func (job *BkDbmonInstall) Init(m *jobruntime.JobGenericRuntime) error {
}
}
for _, svrItem := range job.params.Servers {
if len(svrItem.ServerPorts) >= 0 {
if len(svrItem.ServerPorts) > 0 {
if svrItem.ServerIP == "" {
job.runtime.Logger.Error("BkDbmonInstall Init params validate failed,err:ServerIP is empty")
return fmt.Errorf("ServerIP is empty")
Expand Down
60 changes: 57 additions & 3 deletions dbm-services/redis/db-tools/dbactuator/pkg/atomjobs/atomredis/redis_backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/go-playground/validator/v10"
"github.com/gofrs/flock"
"gopkg.in/yaml.v2"
)

// TendisSSDSetLogCount tendisSSD设置log参数
Expand Down Expand Up @@ -250,8 +251,9 @@ type BackupTask struct {
BackupFile string `json:"backup_file"` // 备份的目标文件,如果文件过大会切割成多个
BackupFileSize int64 `json:"backup_file_size"` // 备份文件大小(已切割 or 已压缩 or 已打包)
BackupTaskID string `json:"backup_taskid"`
BackupMD5 string `json:"backup_md5"` // 目前为空
BackupTag string `json:"backup_tag"` // REDIS_FULL or REDIS_BINLOG
BackupMD5 string `json:"backup_md5"` // 目前为空
BackupTag string `json:"backup_tag"` // REDIS_FULL or REDIS_BINLOG
ShardValue string `json:"shard_value"` // shard值
// 全备尽管会切成多个文件,但其生成的起始时间、结束时间一样
StartTime customtime.CustomTime `json:"start_time"` // 生成全备的起始时间
EndTime customtime.CustomTime `json:"end_time"` // //生成全备的结束时间
Expand Down Expand Up @@ -359,6 +361,10 @@ func (task *BackupTask) GoFullBakcup() {
if task.Err != nil {
return
}
task.getRedisShardVal()
if task.Err != nil {
return
}

// 如果有备份正在执行,则先等待其完成
task.Err = task.Cli.WaitForBackupFinish()
Expand Down Expand Up @@ -469,6 +475,54 @@ func (task *BackupTask) PrecheckDisk() {
task.Addr(), task.DataSize/1024/1024, task.BackupDir, bakDiskUsg.AvailSize/1024/1024))
}

func (task *BackupTask) getRedisShardVal() {
var enabled bool
var masterNode *myredis.ClusterNodeData
enabled, task.Err = task.Cli.IsClusterEnabled()
if task.Err != nil {
return
}
if enabled {
masterNode, task.Err = task.Cli.RedisClusterGetMasterNode(task.Addr())
if task.Err != nil {
return
}
task.ShardValue = masterNode.SlotSrcStr
return
}
if util.FileExists(consts.BkDbmonConfFile) {
confData, err := os.ReadFile(consts.BkDbmonConfFile)
if err != nil {
err = fmt.Errorf("read file(%s) fail,err:%v", consts.BkDbmonConfFile, err)
mylog.Logger.Warn(err.Error())
return
}
if !strings.Contains(string(confData), "server_shards:") {
return
}
type servers struct {
Servers []struct {
ServerShards map[string]string `yaml:"server_shards"`
} `yaml:"servers"`
}
var serversObj servers
err = yaml.Unmarshal(confData, &serversObj)
if err != nil {
err = fmt.Errorf("yaml.Unmarshal fail,err:%v", err)
mylog.Logger.Warn(err.Error())
return
}
for _, server := range serversObj.Servers {
for ipPort, shardVal := range server.ServerShards {
if ipPort == task.Addr() {
task.ShardValue = shardVal
return
}
}
}
}
}

// RedisInstanceBackup redis(cache)实例备份
func (task *BackupTask) RedisInstanceBackup() {
var srcFile string
Expand Down Expand Up @@ -625,7 +679,7 @@ func (task *BackupTask) TendisSSDInstanceBackup() {
mylog.Logger.Error(task.Err.Error())
return
}
task.BackupFile = filepath.Join(task.BackupDir, tarFile)
task.BackupFile = tarFile
task.GetBakFilesSize()
if task.Err != nil {
return
Expand Down
12 changes: 8 additions & 4 deletions dbm-services/redis/db-tools/dbactuator/tests/clustertest/twemproxy_swtich.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,10 @@ func TwemproxyCacheSwitch(serverIP,
SetBkDbmonPkg(bkdbmonPkgName, bkdbmonPkgMd5).
SetDbtoolsPkg(dbtoolsPkgName, dbtoolsPkgMd5).
SetBackupConf().
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum)
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance)
if installTest.Err != nil {
return installTest.Err
}
Expand Down Expand Up @@ -172,8 +174,10 @@ func TwemproxyCacheSwitchRestoreEnv(serverIP,
SetBkDbmonPkg(bkdbmonPkgName, bkdbmonPkgMd5).
SetDbtoolsPkg(dbtoolsPkgName, dbtoolsPkgMd5).
SetBackupConf().
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum)
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance)
if installTest.Err != nil {
return installTest.Err
}
Expand Down
44 changes: 39 additions & 5 deletions dbm-services/redis/db-tools/dbactuator/tests/redistest/bkdbmon_install.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,35 @@ func (test *BkDBmonInstallTest) SetBackupConf() *BkDBmonInstallTest {
return test
}

func getRedisClusterServerShargs(ip string, startPort, instNum int, dbType string) (shards map[string]string) {
if !consts.IsTwemproxyClusterType(dbType) {
return
}
if startPort == 0 {
startPort = consts.TestRedisMasterStartPort
}
if instNum == 0 {
instNum = 4
}
var port int
var instStr string
var segStart int
var segEnd int
shards = make(map[string]string)
segStep := (consts.TwemproxyMaxSegment + 1) / instNum
for i := 0; i < instNum; i++ {
segStart = i * segStep
segEnd = (i+1)*segStep - 1
port = startPort + i
instStr = fmt.Sprintf("%s:%d", ip, port)
shards[instStr] = fmt.Sprintf("%d-%d", segStart, segEnd)
}
return
}

// AppendMasterServer append master server
func (test *BkDBmonInstallTest) AppendMasterServer(masterIP string, startPort, instNum int) *BkDBmonInstallTest {
func (test *BkDBmonInstallTest) AppendMasterServer(masterIP string, startPort, instNum int,
dbType string) *BkDBmonInstallTest {
if test.Err != nil {
return test
}
Expand All @@ -112,10 +139,15 @@ func (test *BkDBmonInstallTest) AppendMasterServer(masterIP string, startPort, i
svrItem := atomredis.ConfServerItem{
BkBizID: "200500194",
BkCloudID: 246,
App: "testapp",
AppName: "测试app",
ClusterDomain: "tendisx.aaaa.testapp.db",
ClusterName: "aaaa",
ClusterType: dbType,
MetaRole: consts.MetaRoleRedisMaster,
ServerIP: masterIP,
ServerPorts: ports,
ServerShards: getRedisClusterServerShargs(masterIP, startPort, instNum, dbType),
}
test.Servers = append(test.Servers, svrItem)
return test
Expand All @@ -139,7 +171,8 @@ func (test *BkDBmonInstallTest) OnlyAEmptyServer(ip string) *BkDBmonInstallTest
}

// AppendSlaveServer append slave server
func (test *BkDBmonInstallTest) AppendSlaveServer(slaveIP string, startPort, instNum int) *BkDBmonInstallTest {
func (test *BkDBmonInstallTest) AppendSlaveServer(slaveIP string, startPort, instNum int,
dbType string) *BkDBmonInstallTest {
if test.Err != nil {
return test
}
Expand All @@ -160,10 +193,11 @@ func (test *BkDBmonInstallTest) AppendSlaveServer(slaveIP string, startPort, ins
AppName: "测试app",
ClusterDomain: "tendisx.aaaa.testapp.db",
ClusterName: "aaaa",
ClusterType: consts.TendisTypePredixyTendisplusCluster,
ClusterType: dbType,
MetaRole: consts.MetaRoleRedisSlave,
ServerIP: slaveIP,
ServerPorts: ports,
ServerShards: getRedisClusterServerShargs(slaveIP, startPort, instNum, dbType),
}
test.Servers = append(test.Servers, svrItem)
return test
Expand Down Expand Up @@ -236,8 +270,8 @@ func BkDbmonInstall(serverIP, dbtoolsPkgName, dbtoolsPkgMd5, bkdbmonPkgName, bkd
SetBkDbmonPkg(bkdbmonPkgName, bkdbmonPkgMd5).
SetDbtoolsPkg(dbtoolsPkgName, dbtoolsPkgMd5).
SetBackupConf().
AppendMasterServer(serverIP, masterStartPort, consts.TestRedisInstanceNum).
AppendSlaveServer(serverIP, slaveStartPort, consts.TestRedisInstanceNum)
AppendMasterServer(serverIP, masterStartPort, consts.TestRedisInstanceNum, dbType).
AppendSlaveServer(serverIP, slaveStartPort, consts.TestRedisInstanceNum, dbType)
if bkdbmonTest.Err != nil {
return
}
Expand Down
3 changes: 0 additions & 3 deletions dbm-services/redis/db-tools/dbactuator/tests/redistest/redis_install.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,9 +404,6 @@ func (test *RedisInstallTest) SetTendisplusRedisConf() {
"scanCntIndexMgr": "10000",
"truncateBinlogIntervalMs": "100",
"minbinlogkeepsec": "1800",
"binlogdelrange": "500000",
"migrate-gc-enabled": "false",
"deletefilesinrange-for-binlog": "1",
"incrpushthreadnum": "10",
"rename-command": `config confxx
rename-command flushdb cleandb
Expand Down
14 changes: 7 additions & 7 deletions dbm-services/redis/db-tools/dbactuator/tests/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
repoUser=""
repoPassword=""

tendisplusPkgName="tendisplus-2.5.0-rocksdb-v6.23.3.tgz"
tendisplusPkgMd5="573fac8917f3cb6d73d4913471a6eacc"
tendisplusPkgName="tendisplus-2.6.0-rocksdb-v6.23.3.tgz"
tendisplusPkgMd5="eaf90d7072740fd232b157d9cb32a425"

redisPkgName="redis-6.2.7.tar.gz"
redisPkgMd5="1fc9e5c3a044ce523844a6f2717e5ac3"
Expand All @@ -17,14 +17,14 @@ tendisssdPkgMd5="7bfe87efbe017c689c3f4a11bb2a8be9"
predixyPkgName="predixy-1.4.0.tar.gz"
predixyPkgMd5="24aba4a96dcf7f8581d2fde89d062455"

twemproxyPkgName="twemproxy-0.4.1-v23.tar.gz"
twemproxyPkgMd5="41850e44bebfce84ebd4d0cf4cce6833"
twemproxyPkgName="twemproxy-0.4.1-v27.tar.gz"
twemproxyPkgMd5="b7fcec49a43da9fdb5acde0a42287d43"

dbtoolsPkgName="dbtools.tar.gz"
dbtoolsPkgMd5="ced0fa280c63cb31536fefc1845f3ff0"

bkdbmonPkgName="bk-dbmon-v0.9.tar.gz"
bkdbmonPkgMd5="a579e2ffd74259f3dd66d23a10a170ba"
bkdbmonPkgName="bk-dbmon-v0.12.tar.gz"
bkdbmonPkgMd5="2a3a51c3b4a7dce4300e894e19f2f0ea"

repoUrl=""

Expand Down Expand Up @@ -144,7 +144,7 @@ if [[ -e $localBkDbmonPkgName ]]; then
localBkDbmonPkgMd5=$(md5sum $localBkDbmonPkgName | awk '{print $1}')
fi

wgetCmd="wget --user=$repoUser --password=$repoPassword $repoUrl/tendisplus/Tendisplus-2.5/$tendisplusPkgName -O $localTendisplusPkgName"
wgetCmd="wget --user=$repoUser --password=$repoPassword $repoUrl/tendisplus/Tendisplus-2.6/$tendisplusPkgName -O $localTendisplusPkgName"
if [[ ! -e $localTendisplusPkgName ]]; then
echo $wgetCmd
$wgetCmd
Expand Down
Loading
Loading