Skip to content

Commit

Permalink
fix(redis): redis 备份上报 添加 segment and slots 等信息 #997
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukemakeit @gaohongsong
lukemakeit authored and gaohongsong committed Sep 12, 2023
1 parent cc5d8cc commit 8a64ebd
Show file tree
Hide file tree
Showing 25 changed files with 532 additions and 111 deletions.
17 changes: 13 additions & 4 deletions dbm-services/redis/db-tools/dbactuator/example/bkdbmon_install.example.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,17 +59,24 @@ bk-dbmon安装:
"bk_cloud_id":"246",
"app":"testapp",
"app_name":"测试app",
"cluster_domain":"tendisx.aaaa.testapp.db",
"cluster_domain":"cache.aaaa.testapp.db",
"cluster_name":"aaaa",
"cluster_type":"PredixyTendisplusCluster",
"cluster_type":"TwemproxyRedisInstance",
"meta_role":"redis_master",
"server_ip":"127.0.0.1",
"server_ports":[
30000,
30001,
30002,
30003
]
],
"server_shards":{
"a.a.a.a:12000":"0-104999",
"a.a.a.a:12001":"105000-209999",
"a.a.a.a:12002":"210000-314999",
"a.a.a.a:12003":"315000-419999"
},
"cache_backup_mode":"rdb"
},
{
"bk_biz_id":"200500194",
Expand All @@ -86,7 +93,9 @@ bk-dbmon安装:
31001,
31002,
31003
]
],
"server_shards":{},
"cache_backup_mode":""
}
]
}
Expand Down
34 changes: 32 additions & 2 deletions dbm-services/redis/db-tools/dbactuator/models/myredis/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -712,8 +712,8 @@ func (db *RedisClient) Sscan(keyname string, cursor uint64, match string, count
fields, retCursor, err = db.InstanceClient.SScan(context.TODO(), keyname, cursor, match, count).Result()
}
if err != nil && err != redis.Nil {
mylog.Logger.Error("Redis 'sscan %s %d match %s count %s' command fail,err:%v,addr:%s", keyname, cursor, match, count,
err, db.Addr)
mylog.Logger.Error("Redis 'sscan %s %d match %s count %d' command fail,err:%v,addr:%s",
keyname, cursor, match, count, err, db.Addr)
return fields, 0, err
}
return fields, retCursor, nil
Expand Down Expand Up @@ -1761,3 +1761,33 @@ func (db *RedisClient) GetClusterNodesStr() (ret string, err error) {
}
return
}

// RedisClusterGetMasterNode 获取master节点信息(如果 addr是master则返回它的node信息,否则找到它的masterID,进而找到master的node信息)
func (db *RedisClient) RedisClusterGetMasterNode(addr string) (masterNode *ClusterNodeData, err error) {
addrToNodes, err := db.GetAddrMapToNodes()
if err != nil {
return
}
myNode, ok := addrToNodes[addr]
if !ok {
err = fmt.Errorf("addr:%s not found in cluster nodes", addr)
mylog.Logger.Error(err.Error())
return
}
if myNode.GetRole() == consts.RedisMasterRole {
masterNode = myNode
return
}
masterNodeID := myNode.MasterID
idToNode, err := db.GetNodeIDMapToNodes()
if err != nil {
return
}
masterNode, ok = idToNode[masterNodeID]
if !ok {
err = fmt.Errorf("masterNodeID:%s not found in cluster nodes", masterNodeID)
mylog.Logger.Error(err.Error())
return
}
return
}
26 changes: 14 additions & 12 deletions dbm-services/redis/db-tools/dbactuator/pkg/atomjobs/atomredis/bkdbmon_install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,19 @@ import (

// ConfServerItem servers配置项
type ConfServerItem struct {
BkBizID string `json:"bk_biz_id" yaml:"bk_biz_id" validate:"required"`
BkCloudID int64 `json:"bk_cloud_id" yaml:"bk_cloud_id"`
App string `json:"app" yaml:"app" validate:"required"`
AppName string `json:"app_name" yaml:"app_name" validate:"required"`
ClusterDomain string `json:"cluster_domain" yaml:"cluster_domain" validate:"required"`
ClusterName string `json:"cluster_name" yaml:"cluster_name" validate:"required"`
ClusterType string `json:"cluster_type" yaml:"cluster_type" validate:"required"`
MetaRole string `json:"meta_role" yaml:"meta_role" validate:"required"`
ServerIP string `json:"server_ip" yaml:"server_ip" validate:"required"`
ServerPorts []int `json:"server_ports" yaml:"server_ports" validate:"required"`
Shard string `json:"shard" yaml:"shard"`
BkBizID string `json:"bk_biz_id" yaml:"bk_biz_id" validate:"required"`
BkCloudID int64 `json:"bk_cloud_id" yaml:"bk_cloud_id"`
App string `json:"app" yaml:"app" validate:"required"`
AppName string `json:"app_name" yaml:"app_name" validate:"required"`
ClusterDomain string `json:"cluster_domain" yaml:"cluster_domain" validate:"required"`
ClusterName string `json:"cluster_name" yaml:"cluster_name" validate:"required"`
ClusterType string `json:"cluster_type" yaml:"cluster_type" validate:"required"`
MetaRole string `json:"meta_role" yaml:"meta_role" validate:"required"`
ServerIP string `json:"server_ip" yaml:"server_ip" validate:"required"`
ServerPorts []int `json:"server_ports" yaml:"server_ports" validate:"required"`
ServerShards map[string]string `json:"server_shards" yaml:"server_shards"`
CacheBackupMode string `json:"cache_backup_mode" yaml:"cache_backup_mode"` // aof or rdb
Shard string `json:"shard" yaml:"shard"`
}

// BkDbmonInstallParams 安装参数
Expand Down Expand Up @@ -90,7 +92,7 @@ func (job *BkDbmonInstall) Init(m *jobruntime.JobGenericRuntime) error {
}
}
for _, svrItem := range job.params.Servers {
if len(svrItem.ServerPorts) >= 0 {
if len(svrItem.ServerPorts) > 0 {
if svrItem.ServerIP == "" {
job.runtime.Logger.Error("BkDbmonInstall Init params validate failed,err:ServerIP is empty")
return fmt.Errorf("ServerIP is empty")
Expand Down
60 changes: 57 additions & 3 deletions dbm-services/redis/db-tools/dbactuator/pkg/atomjobs/atomredis/redis_backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/go-playground/validator/v10"
"github.com/gofrs/flock"
"gopkg.in/yaml.v2"
)

// TendisSSDSetLogCount tendisSSD设置log参数
Expand Down Expand Up @@ -250,8 +251,9 @@ type BackupTask struct {
BackupFile string `json:"backup_file"` // 备份的目标文件,如果文件过大会切割成多个
BackupFileSize int64 `json:"backup_file_size"` // 备份文件大小(已切割 or 已压缩 or 已打包)
BackupTaskID string `json:"backup_taskid"`
BackupMD5 string `json:"backup_md5"` // 目前为空
BackupTag string `json:"backup_tag"` // REDIS_FULL or REDIS_BINLOG
BackupMD5 string `json:"backup_md5"` // 目前为空
BackupTag string `json:"backup_tag"` // REDIS_FULL or REDIS_BINLOG
ShardValue string `json:"shard_value"` // shard值
// 全备尽管会切成多个文件,但其生成的起始时间、结束时间一样
StartTime customtime.CustomTime `json:"start_time"` // 生成全备的起始时间
EndTime customtime.CustomTime `json:"end_time"` // //生成全备的结束时间
Expand Down Expand Up @@ -359,6 +361,10 @@ func (task *BackupTask) GoFullBakcup() {
if task.Err != nil {
return
}
task.getRedisShardVal()
if task.Err != nil {
return
}

// 如果有备份正在执行,则先等待其完成
task.Err = task.Cli.WaitForBackupFinish()
Expand Down Expand Up @@ -469,6 +475,54 @@ func (task *BackupTask) PrecheckDisk() {
task.Addr(), task.DataSize/1024/1024, task.BackupDir, bakDiskUsg.AvailSize/1024/1024))
}

func (task *BackupTask) getRedisShardVal() {
var enabled bool
var masterNode *myredis.ClusterNodeData
enabled, task.Err = task.Cli.IsClusterEnabled()
if task.Err != nil {
return
}
if enabled {
masterNode, task.Err = task.Cli.RedisClusterGetMasterNode(task.Addr())
if task.Err != nil {
return
}
task.ShardValue = masterNode.SlotSrcStr
return
}
if util.FileExists(consts.BkDbmonConfFile) {
confData, err := os.ReadFile(consts.BkDbmonConfFile)
if err != nil {
err = fmt.Errorf("read file(%s) fail,err:%v", consts.BkDbmonConfFile, err)
mylog.Logger.Warn(err.Error())
return
}
if !strings.Contains(string(confData), "server_shards:") {
return
}
type servers struct {
Servers []struct {
ServerShards map[string]string `yaml:"server_shards"`
} `yaml:"servers"`
}
var serversObj servers
err = yaml.Unmarshal(confData, &serversObj)
if err != nil {
err = fmt.Errorf("yaml.Unmarshal fail,err:%v", err)
mylog.Logger.Warn(err.Error())
return
}
for _, server := range serversObj.Servers {
for ipPort, shardVal := range server.ServerShards {
if ipPort == task.Addr() {
task.ShardValue = shardVal
return
}
}
}
}
}

// RedisInstanceBackup redis(cache)实例备份
func (task *BackupTask) RedisInstanceBackup() {
var srcFile string
Expand Down Expand Up @@ -625,7 +679,7 @@ func (task *BackupTask) TendisSSDInstanceBackup() {
mylog.Logger.Error(task.Err.Error())
return
}
task.BackupFile = filepath.Join(task.BackupDir, tarFile)
task.BackupFile = tarFile
task.GetBakFilesSize()
if task.Err != nil {
return
Expand Down
12 changes: 8 additions & 4 deletions dbm-services/redis/db-tools/dbactuator/tests/clustertest/twemproxy_swtich.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,10 @@ func TwemproxyCacheSwitch(serverIP,
SetBkDbmonPkg(bkdbmonPkgName, bkdbmonPkgMd5).
SetDbtoolsPkg(dbtoolsPkgName, dbtoolsPkgMd5).
SetBackupConf().
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum)
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance)
if installTest.Err != nil {
return installTest.Err
}
Expand Down Expand Up @@ -172,8 +174,10 @@ func TwemproxyCacheSwitchRestoreEnv(serverIP,
SetBkDbmonPkg(bkdbmonPkgName, bkdbmonPkgMd5).
SetDbtoolsPkg(dbtoolsPkgName, dbtoolsPkgMd5).
SetBackupConf().
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum)
AppendMasterServer(serverIP, consts.TestRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance).
AppendMasterServer(serverIP, consts.TestSyncRedisMasterStartPort, consts.TestRedisInstanceNum,
consts.TendisTypeTwemproxyRedisInstance)
if installTest.Err != nil {
return installTest.Err
}
Expand Down
44 changes: 39 additions & 5 deletions dbm-services/redis/db-tools/dbactuator/tests/redistest/bkdbmon_install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,35 @@ func (test *BkDBmonInstallTest) SetBackupConf() *BkDBmonInstallTest {
return test
}

func getRedisClusterServerShargs(ip string, startPort, instNum int, dbType string) (shards map[string]string) {
if !consts.IsTwemproxyClusterType(dbType) {
return
}
if startPort == 0 {
startPort = consts.TestRedisMasterStartPort
}
if instNum == 0 {
instNum = 4
}
var port int
var instStr string
var segStart int
var segEnd int
shards = make(map[string]string)
segStep := (consts.TwemproxyMaxSegment + 1) / instNum
for i := 0; i < instNum; i++ {
segStart = i * segStep
segEnd = (i+1)*segStep - 1
port = startPort + i
instStr = fmt.Sprintf("%s:%d", ip, port)
shards[instStr] = fmt.Sprintf("%d-%d", segStart, segEnd)
}
return
}

// AppendMasterServer append master server
func (test *BkDBmonInstallTest) AppendMasterServer(masterIP string, startPort, instNum int) *BkDBmonInstallTest {
func (test *BkDBmonInstallTest) AppendMasterServer(masterIP string, startPort, instNum int,
dbType string) *BkDBmonInstallTest {
if test.Err != nil {
return test
}
Expand All @@ -112,10 +139,15 @@ func (test *BkDBmonInstallTest) AppendMasterServer(masterIP string, startPort, i
svrItem := atomredis.ConfServerItem{
BkBizID: "200500194",
BkCloudID: 246,
App: "testapp",
AppName: "测试app",
ClusterDomain: "tendisx.aaaa.testapp.db",
ClusterName: "aaaa",
ClusterType: dbType,
MetaRole: consts.MetaRoleRedisMaster,
ServerIP: masterIP,
ServerPorts: ports,
ServerShards: getRedisClusterServerShargs(masterIP, startPort, instNum, dbType),
}
test.Servers = append(test.Servers, svrItem)
return test
Expand All @@ -139,7 +171,8 @@ func (test *BkDBmonInstallTest) OnlyAEmptyServer(ip string) *BkDBmonInstallTest
}

// AppendSlaveServer append slave server
func (test *BkDBmonInstallTest) AppendSlaveServer(slaveIP string, startPort, instNum int) *BkDBmonInstallTest {
func (test *BkDBmonInstallTest) AppendSlaveServer(slaveIP string, startPort, instNum int,
dbType string) *BkDBmonInstallTest {
if test.Err != nil {
return test
}
Expand All @@ -160,10 +193,11 @@ func (test *BkDBmonInstallTest) AppendSlaveServer(slaveIP string, startPort, ins
AppName: "测试app",
ClusterDomain: "tendisx.aaaa.testapp.db",
ClusterName: "aaaa",
ClusterType: consts.TendisTypePredixyTendisplusCluster,
ClusterType: dbType,
MetaRole: consts.MetaRoleRedisSlave,
ServerIP: slaveIP,
ServerPorts: ports,
ServerShards: getRedisClusterServerShargs(slaveIP, startPort, instNum, dbType),
}
test.Servers = append(test.Servers, svrItem)
return test
Expand Down Expand Up @@ -236,8 +270,8 @@ func BkDbmonInstall(serverIP, dbtoolsPkgName, dbtoolsPkgMd5, bkdbmonPkgName, bkd
SetBkDbmonPkg(bkdbmonPkgName, bkdbmonPkgMd5).
SetDbtoolsPkg(dbtoolsPkgName, dbtoolsPkgMd5).
SetBackupConf().
AppendMasterServer(serverIP, masterStartPort, consts.TestRedisInstanceNum).
AppendSlaveServer(serverIP, slaveStartPort, consts.TestRedisInstanceNum)
AppendMasterServer(serverIP, masterStartPort, consts.TestRedisInstanceNum, dbType).
AppendSlaveServer(serverIP, slaveStartPort, consts.TestRedisInstanceNum, dbType)
if bkdbmonTest.Err != nil {
return
}
Expand Down
3 changes: 0 additions & 3 deletions dbm-services/redis/db-tools/dbactuator/tests/redistest/redis_install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,9 +404,6 @@ func (test *RedisInstallTest) SetTendisplusRedisConf() {
"scanCntIndexMgr": "10000",
"truncateBinlogIntervalMs": "100",
"minbinlogkeepsec": "1800",
"binlogdelrange": "500000",
"migrate-gc-enabled": "false",
"deletefilesinrange-for-binlog": "1",
"incrpushthreadnum": "10",
"rename-command": `config confxx
rename-command flushdb cleandb
Expand Down
14 changes: 7 additions & 7 deletions dbm-services/redis/db-tools/dbactuator/tests/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
repoUser=""
repoPassword=""

tendisplusPkgName="tendisplus-2.5.0-rocksdb-v6.23.3.tgz"
tendisplusPkgMd5="573fac8917f3cb6d73d4913471a6eacc"
tendisplusPkgName="tendisplus-2.6.0-rocksdb-v6.23.3.tgz"
tendisplusPkgMd5="eaf90d7072740fd232b157d9cb32a425"

redisPkgName="redis-6.2.7.tar.gz"
redisPkgMd5="1fc9e5c3a044ce523844a6f2717e5ac3"
Expand All @@ -17,14 +17,14 @@ tendisssdPkgMd5="7bfe87efbe017c689c3f4a11bb2a8be9"
predixyPkgName="predixy-1.4.0.tar.gz"
predixyPkgMd5="24aba4a96dcf7f8581d2fde89d062455"

twemproxyPkgName="twemproxy-0.4.1-v23.tar.gz"
twemproxyPkgMd5="41850e44bebfce84ebd4d0cf4cce6833"
twemproxyPkgName="twemproxy-0.4.1-v27.tar.gz"
twemproxyPkgMd5="b7fcec49a43da9fdb5acde0a42287d43"

dbtoolsPkgName="dbtools.tar.gz"
dbtoolsPkgMd5="ced0fa280c63cb31536fefc1845f3ff0"

bkdbmonPkgName="bk-dbmon-v0.9.tar.gz"
bkdbmonPkgMd5="a579e2ffd74259f3dd66d23a10a170ba"
bkdbmonPkgName="bk-dbmon-v0.12.tar.gz"
bkdbmonPkgMd5="2a3a51c3b4a7dce4300e894e19f2f0ea"

repoUrl=""

Expand Down Expand Up @@ -144,7 +144,7 @@ if [[ -e $localBkDbmonPkgName ]]; then
localBkDbmonPkgMd5=$(md5sum $localBkDbmonPkgName | awk '{print $1}')
fi

wgetCmd="wget --user=$repoUser --password=$repoPassword $repoUrl/tendisplus/Tendisplus-2.5/$tendisplusPkgName -O $localTendisplusPkgName"
wgetCmd="wget --user=$repoUser --password=$repoPassword $repoUrl/tendisplus/Tendisplus-2.6/$tendisplusPkgName -O $localTendisplusPkgName"
if [[ ! -e $localTendisplusPkgName ]]; then
echo $wgetCmd
$wgetCmd
Expand Down
Loading

0 comments on commit 8a64ebd

Please sign in to comment.