merge: from v1.3.0 to v1.4.0

.github/CODEOWNERS

@@ -1,19 +1,19 @@
 # public
-/.github @zhangzhw8 @gaohongsong @iSecloud
-/docs @zhangzhw8 @gaohongsong @iSecloud
-/*.md @zhangzhw8 @gaohongsong @iSecloud
-/helm-charts @zhangzhw8 @gaohongsong @iSecloud
+/.github @zhangzhw8 @iSecloud
+/docs @zhangzhw8 @iSecloud
+/*.md @zhangzhw8 @iSecloud
+/helm-charts @zhangzhw8 @iSecloud
 
 # dbm-ui
-/dbm-ui @zhangzhw8 @gaohongsong @iSecloud
+/dbm-ui @zhangzhw8 @iSecloud
 /dbm-ui/frontend @hLinx @jinquantianxia
 
 # dbm-services common
 /dbm-services/common @seanlook @xfwduke @lukemakeit @xiepaup @ymakedaq
 /dbm-services/common/db-config @seanlook @xfwduke
 /dbm-services/common/db-resource @ymakedaq @seanlook @xfwduke
 /dbm-services/common/db-dns @omg-by @xiepaup @lukemakeit
-/dbm-services/common/dbha @zyqlzr @xjxia
+/dbm-services/common/dbha @xjxia
 
 # bigdata
 /dbm-services/bigdata @zhangrq5 @zvictorino @wangyao963
@@ -23,7 +23,7 @@
 
 # mysql
 /dbm-services/mysql @seanlook @xfwduke @yksitu @ymakedaq
-/dbm-services/mysql/db-partition @fanfanyangyang @xfwduke @seanlook
+/dbm-services/mysql/db-partition @fanfanyangyang @xfan0805
 /dbm-services/mysql/db-priv @fanfanyangyang @xfwduke @seanlook
 /dbm-services/mysql/db-remote-service @xfwduke @seanlook
 /dbm-services/mysql/db-simulation @seanlook @xfwduke @ymakedaq

dbm-services/common/dbha/ha-module/bk-dbha/templates/configmap.yaml

@@ -10,6 +10,7 @@ data:
       active_db_type: [
         "tendbha:backend",
         "tendbha:proxy",
+        "riak"
       ]
       city: "3"
       campus: "深圳"
@@ -57,6 +58,8 @@ data:
         proxy_pass: "proxy-conn-pass"
         timeout: 10
       redis:
+      riak:
+        timeout: 10
     dns:
       bind_conf:
         host: "bind-api-host"

dbm-services/common/dbha/ha-module/ha.yaml

@@ -9,6 +9,7 @@ agent_conf:
   active_db_type: [
     "tendbha:backend",
     "tendbha:proxy",
+    "riak"
   ]
   city: "3"
   campus: "深圳"
@@ -59,6 +60,8 @@ db_conf:
     timeout: 10
   redis:
     timeout: 10
+  riak:
+    timeout: 10
 password_conf:
   host: "bind-api-host"
   port: 80

dbm-services/mysql/db-partition/handler/handler.go

@@ -179,6 +179,26 @@ func DisablePartition(r *gin.Context) {
 	return
 }
 
+// DisablePartitionByCluster 用于集群禁用时停止分区，标志为 offlinewithclu
+func DisablePartitionByCluster(r *gin.Context) {
+	var input service.DisablePartitionInput
+	if err := r.ShouldBind(&input); err != nil {
+		err = errno.ErrReadEntity.Add(err.Error())
+		slog.Error(err.Error())
+		SendResponse(r, err, nil)
+		return
+	}
+	slog.Info(fmt.Sprintf("ids: %v, operator: %s", input.Ids, input.Operator))
+	err := input.DisablePartitionConfigByCluster()
+	if err != nil {
+		slog.Error(err.Error())
+		SendResponse(r, errors.New(fmt.Sprintf("分区禁用失败!%s", err.Error())), nil)
+		return
+	}
+	SendResponse(r, nil, "分区禁用成功！")
+	return
+}
+
 // EnablePartition TODO
 func EnablePartition(r *gin.Context) {
 	var input service.EnablePartitionInput
@@ -199,6 +219,26 @@ func EnablePartition(r *gin.Context) {
 	return
 }
 
+// EnablePartitionByCluster 集群启用时启用分区
+func EnablePartitionByCluster(r *gin.Context) {
+	var input service.EnablePartitionInput
+	if err := r.ShouldBind(&input); err != nil {
+		err = errno.ErrReadEntity.Add(err.Error())
+		slog.Error(err.Error())
+		SendResponse(r, err, nil)
+		return
+	}
+	slog.Info(fmt.Sprintf("ids: %v, operator: %s", input.Ids, input.Operator))
+	err := input.EnablePartitionByCluster()
+	if err != nil {
+		slog.Error(err.Error())
+		SendResponse(r, errors.New(fmt.Sprintf("分区启用失败!%s", err.Error())), nil)
+		return
+	}
+	SendResponse(r, nil, "分区启用成功！")
+	return
+}
+
 // UpdatePartitionsConfig TODO
 func UpdatePartitionsConfig(r *gin.Context) {
 	var input service.CreatePartitionsInput

dbm-services/mysql/db-partition/main.go

@@ -4,17 +4,17 @@ import (
 	"net/http"
 	"os"
 
-	"dbm-services/common/go-pubpkg/apm/metric"
-	"dbm-services/common/go-pubpkg/apm/trace"
-	"dbm-services/mysql/db-partition/monitor"
-
 	"github.com/gin-gonic/gin"
 	"github.com/golang-migrate/migrate/v4"
 	flag "github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
 	"golang.org/x/exp/slog"
 
+	"dbm-services/common/go-pubpkg/apm/metric"
+	"dbm-services/common/go-pubpkg/apm/trace"
+	"dbm-services/mysql/db-partition/monitor"
+
 	"dbm-services/mysql/db-partition/assests"
 	"dbm-services/mysql/db-partition/cron"
 	"dbm-services/mysql/db-partition/model"

dbm-services/mysql/db-partition/router/router.go

@@ -21,6 +21,8 @@ func RegisterRouter(engine *gin.Engine) {
 	p.POST("/dry_run", handler.DryRun)
 	p.POST("/disable_partition", handler.DisablePartition)
 	p.POST("/enable_partition", handler.EnablePartition)
+	p.POST("/disable_partition_cluster", handler.DisablePartitionByCluster)
+	p.POST("/enable_partition_cluster", handler.EnablePartitionByCluster)
 	// 更新分区配置
 	p.POST("/update_conf", handler.UpdatePartitionsConfig)
 	p.POST("/create_log", handler.CreatePartitionLog)

dbm-services/mysql/db-partition/service/check_partition_base_func.go

@@ -56,6 +56,8 @@ func (config *PartitionConfig) GetPartitionDbLikeTbLike(dbtype string, splitCnt
 				}
 				AddString(&addSqls, sql)
 				if tb.Phase == online {
+					// 启用的分区规则，会执行删除历史分区
+					// 禁用的分区规则，会新增分区，但是不会删除历史分区
 					sql, err = tb.GetDropPartitionSql()
 					if err != nil {
 						slog.Error("msg", "GetDropPartitionSql error", err)
@@ -642,7 +644,7 @@ func CreatePartitionTicket(check Checker, objects []PartitionObject, zoneOffset
 		zone, date, scheduler, "", ExecuteAsynchronous, check.ClusterType)
 }
 
-// NeedPartition TODO
+// NeedPartition 获取需要实施的分区规则
 func NeedPartition(cronType string, clusterType string, zoneOffset int, cronDate string) ([]*Checker, error) {
 	var configTb, logTb string
 	var all, doNothing []*Checker
@@ -657,10 +659,11 @@ func NeedPartition(cronType string, clusterType string, zoneOffset int, cronDate
 		return nil, errors.New("不支持的db类型")
 	}
 	vzone := fmt.Sprintf("%+03d:00", zoneOffset)
+	// 集群被offline时，其分区规则也被禁用，规则不会被定时任务执行
 	vsql := fmt.Sprintf(
 		"select id as config_id, bk_biz_id, cluster_id, immute_domain, port, bk_cloud_id,"+
-			" '%s' as cluster_type from `%s`.`%s` where time_zone='%s' order by 2,3;",
-		clusterType, viper.GetString("db.name"), configTb, vzone)
+			" '%s' as cluster_type from `%s`.`%s` where time_zone='%s' and phase in ('%s','%s') order by 2,3;",
+		clusterType, viper.GetString("db.name"), configTb, vzone, online, offline)
 	slog.Info(vsql)
 	err := model.DB.Self.Raw(vsql).Scan(&all).Error
 	if err != nil {

dbm-services/mysql/db-partition/service/manage_config.go

@@ -489,6 +489,42 @@ func (m *DisablePartitionInput) DisablePartitionConfig() error {
 	return nil
 }
 
+// DisablePartitionConfigByCluster TODO
+func (m *DisablePartitionInput) DisablePartitionConfigByCluster() error {
+	if len(m.ClusterIds) == 0 {
+		return errno.ConfigIdIsEmpty
+	}
+	var tbName string
+	// 判断是mysql集群还是spider集群
+	var logTbName string
+	switch strings.ToLower(m.ClusterType) {
+	case Tendbha, Tendbsingle:
+		tbName = MysqlPartitionConfig
+		logTbName = MysqlManageLogsTable
+	case Tendbcluster:
+		tbName = SpiderPartitionConfig
+		logTbName = SpiderManageLogsTable
+	default:
+		return errors.New("不支持的db类型")
+	}
+	var list []string
+	for _, item := range m.ClusterIds {
+		list = append(list, strconv.FormatInt(int64(item), 10))
+
+	}
+	db := model.DB.Self.Table(tbName)
+	result := db.
+		Where(fmt.Sprintf("cluster_id in (%s)", strings.Join(list, ","))).
+		Update("phase", offlinewithclu)
+	if result.Error != nil {
+		return result.Error
+	}
+	for _, id := range m.Ids {
+		CreateManageLog(tbName, logTbName, id, "DisableByCluster", m.Operator)
+	}
+	return nil
+}
+
 // EnablePartitionConfig TODO
 func (m *EnablePartitionInput) EnablePartitionConfig() error {
 	if len(m.Ids) == 0 {
@@ -525,6 +561,42 @@ func (m *EnablePartitionInput) EnablePartitionConfig() error {
 	return nil
 }
 
+// EnablePartitionByCluster TODO
+func (m *EnablePartitionInput) EnablePartitionByCluster() error {
+	if len(m.ClusterIds) == 0 {
+		return errno.ConfigIdIsEmpty
+	}
+	var tbName string
+	// 判断是mysql集群还是spider集群
+	var logTbName string
+	switch strings.ToLower(m.ClusterType) {
+	case Tendbha, Tendbsingle:
+		tbName = MysqlPartitionConfig
+		logTbName = MysqlManageLogsTable
+	case Tendbcluster:
+		tbName = SpiderPartitionConfig
+		logTbName = SpiderManageLogsTable
+	default:
+		return errors.New("不支持的db类型")
+	}
+	var list []string
+	for _, item := range m.ClusterIds {
+		list = append(list, strconv.FormatInt(int64(item), 10))
+
+	}
+	db := model.DB.Self.Table(tbName)
+	result := db.
+		Where(fmt.Sprintf("cluster_id in (%s)", strings.Join(list, ","))).
+		Update("phase", online)
+	if result.Error != nil {
+		return result.Error
+	}
+	for _, id := range m.Ids {
+		CreateManageLog(tbName, logTbName, id, "EnableByCluster", m.Operator)
+	}
+	return nil
+}
+
 func (m *CreatePartitionsInput) compareWithSameArray() (warnings []string, err error) {
 	l := len(m.DbLikes)
 	for i := 0; i < l; i++ {

dbm-services/mysql/db-partition/service/manage_config_object.go

@@ -16,6 +16,7 @@ const SpiderPartitionCronLogTable = "spider_partition_cron_log"
 
 const online = "online"
 const offline = "offline"
+const offlinewithclu = "offlinewithclu"
 const extraTime = 15
 
 // MysqlManageLogsTable TODO
@@ -99,13 +100,15 @@ type DisablePartitionInput struct {
 	ClusterType string `json:"cluster_type"`
 	Operator    string `json:"operator"`
 	Ids         []int  `json:"ids"`
+	ClusterIds  []int  `json:"cluster_ids"`
 }
 
 // EnablePartitionInput TODO
 type EnablePartitionInput struct {
 	ClusterType string `json:"cluster_type"`
 	Operator    string `json:"operator"`
 	Ids         []int  `json:"ids"`
+	ClusterIds  []int  `json:"cluster_ids"`
 }
 
 // ManageLog 审计分区管理行为

dbm-services/mysql/db-priv/service/accout_rule.go

@@ -49,6 +49,10 @@ func (m *BkBizId) QueryAccountRule() ([]*AccountRuleSplitUser, int64, error) {
 		if err != nil {
 			return nil, count, err
 		}
+		// 没有查到帐号规则
+		if len(acountList) == 0 {
+			return nil, count, nil
+		}
 		for _, id := range acountList {
 			accountIds = fmt.Sprintf("%d,%s", id.AccountId, accountIds)
 		}

dbm-services/mysql/db-priv/service/generate_random_string.go

@@ -14,7 +14,7 @@ import (
 const lowercase = "abcdefghijklmnopqrstuvwxyz"
 const uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 const number = "0123456789"
-const symbol = `!#$%&()*+,-./:;<=>?@[]^_{|}~` // 剔除 " ' ` \
+const symbol = `!#%&()*+,-./;<=>?[]^_{|}~` // 剔除@ : $ " ' ` \
 
 // 为密码池添加连续的字母序，数字序，特殊字符序和键盘序
 const continuousSymbols = "~!@#$%^&*()_+"

dbm-services/mysql/db-tools/mysql-monitor/cmd/init.go

@@ -58,7 +58,12 @@ func initLogger(cfg *config.LogConfig) {
 			}
 		}
 
-		ioWriters = append(ioWriters, &lumberjack.Logger{Filename: logFile})
+		ioWriters = append(ioWriters, &lumberjack.Logger{
+			Filename: logFile,
+			MaxAge:   2,
+			//MaxBackups: 2,
+			Compress: true,
+		})
 	}
 
 	handleOpt := slog.HandlerOptions{AddSource: cfg.Source}

dbm-services/mysql/db-tools/mysql-monitor/config2sql.pl

@@ -10,6 +10,8 @@
     my $item_value = encode_json($item);
     $item_value =~ s/"enable":"1"/"enable":true/;
     $item_value =~ s/"enable":"0"/"enable":false/;
+    $item_value =~ s/"enable":""/"enable":false/;
+
     my $sql = sprintf(q#REPLACE INTO 
     tb_config_name_def(
         namespace, conf_type, conf_file, conf_name,