Skip to content

Commit

Permalink
feat(dbm-services): 动态加载mysql pod 启动参数 TencentBlueKing#8218
Browse files Browse the repository at this point in the history
  • Loading branch information
@ymakedaq
ymakedaq committed Nov 27, 2024
1 parent ec0f506 commit d3213ee
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 168 deletions.
107 changes: 77 additions & 30 deletions dbm-services/mysql/db-simulation/app/service/kubernets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,18 +122,6 @@ func (k *DbPodSets) getCreateClusterSqls() []string {
// getClusterPodContanierSpec create cluster pod container spec
// nolint
func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Container {
mysqldStartArgs := []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=20000",
"--max_allowed_packet=1073741824",
"--sql-mode=",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
if cmutil.MySQLVersionParse(mysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") {
mysqldStartArgs = append(mysqldStartArgs, "--default-authentication-plugin=mysql_native_password")
}
return []v1.Container{
{
Name: "backend",
Expand All @@ -144,7 +132,7 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
Resources: k.getResourceLimit(),
ImagePullPolicy: v1.PullIfNotPresent,
Image: k.DbImage,
Args: mysqldStartArgs,
Args: k.getbackendStartArgs(mysqlVersion),
ReadinessProbe: &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Expand All @@ -163,14 +151,7 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
Resources: k.getResourceLimit(),
ImagePullPolicy: v1.PullIfNotPresent,
Image: k.SpiderImage,
Args: []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=25000",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"},
Args: k.getSpiderStartArgs(),
ReadinessProbe: &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Expand All @@ -190,11 +171,7 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
Resources: k.gettdbctlResourceLimit(),
ImagePullPolicy: v1.PullIfNotPresent,
Image: k.TdbCtlImage,
Args: []string{"mysqld", "--defaults-file=/etc/my.cnf", "--port=26000", "--tc-admin=1",
"--dbm-allow-standalone-primary",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"},
Args: k.getTdbctlStartArgs(),
ReadinessProbe: &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Expand All @@ -208,6 +185,65 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
}
}

func (k *DbPodSets) getTdbctlStartArgs() (args []string) {
args = []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--port=26000",
"--tc-admin=1",
"--dbm-allow-standalone-primary",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
dbArgs, err := model.GetStartArsg("tdbctl", LatestVersion)
if err != nil {
logger.Warn("get tdbctl start args failed %s", err.Error())
return
}
args = append(args, strings.Split(dbArgs, " ")...)
return
}

func (k *DbPodSets) getSpiderStartArgs() (args []string) {
args = []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=25000",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
dbArgs, err := model.GetStartArsg("spider", LatestVersion)
if err != nil {
logger.Warn("get spider start args failed %s", err.Error())
return
}
args = append(args, strings.Split(dbArgs, " ")...)
return
}

func (k *DbPodSets) getbackendStartArgs(mysqlVersion string) (args []string) {
args = []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=20000",
"--max_allowed_packet=1073741824",
"--sql-mode=",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
if cmutil.MySQLVersionParse(mysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") {
args = append(args, "--default-authentication-plugin=mysql_native_password")
}
dbArgs, err := model.GetStartArsg("mysql", mysqlVersion)
if err != nil {
logger.Warn("get mysql start args failed %s", err.Error())
return
}
args = append(args, strings.Split(dbArgs, " ")...)
return
}

// CreateClusterPod create tendbcluster simulation pod
func (k *DbPodSets) CreateClusterPod(mySQLVersion string) (err error) {
c := &v1.Pod{
Expand Down Expand Up @@ -356,16 +392,27 @@ func (k *DbPodSets) gettdbctlResourceLimit() v1.ResourceRequirements {
return v1.ResourceRequirements{}
}

// CreateMySQLPod create mysql pod
func (k *DbPodSets) CreateMySQLPod(mysqlVersion string) (err error) {
startArgs := []string{
func (k *DbPodSets) getTendbhaPodStartArgs(mysqlVersion string) (args []string) {
args = []string{
"--defaults-file=/etc/my.cnf",
"--skip-log-bin",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s", k.BaseInfo.Charset)}
if cmutil.MySQLVersionParse(mysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") {
startArgs = append(startArgs, "--default-authentication-plugin=mysql_native_password")
args = append(args, "--default-authentication-plugin=mysql_native_password")
}
dbArgs, err := model.GetStartArsg("mysql", mysqlVersion)
if err != nil {
logger.Warn("get mysql start args failed %s", err.Error())
return
}
args = append(args, strings.Split(dbArgs, " ")...)
return
}

// CreateMySQLPod create mysql pod
func (k *DbPodSets) CreateMySQLPod(mysqlVersion string) (err error) {
startArgs := k.getTendbhaPodStartArgs(mysqlVersion)
startArgs = append(startArgs, k.BaseInfo.Args...)
startArgs = append(startArgs, "--user=mysql")
logger.Info("start pod args %v", startArgs)
Expand Down
Empty file added dbm-services/mysql/db-simulation/assets/migrations/000004_add_pod_start_args_table.down.sql
View file
Empty file.
10 changes: 10 additions & 0 deletions dbm-services/mysql/db-simulation/assets/migrations/000004_add_pod_start_args_table.up.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS `tb_mysql_pod_start_cfgs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`component_type` varchar(64) NOT NULL,
`version` varchar(64) NOT NULL,
`start_args` varchar(1024) NOT NULL,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uk_cv` (`component_type`, `version`)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
31 changes: 31 additions & 0 deletions dbm-services/mysql/db-simulation/model/tb_mysql_pod_start_cfg.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
* Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
* Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://opensource.org/licenses/MIT
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

package model

import "time"

// TbMySQLPodStartCfg 模拟执行 pod 启动参数
type TbMySQLPodStartCfg struct {
ID int `gorm:"primaryKey;column:id;type:int(11);not null" json:"-"`
ComponentType string `gorm:"unique;column:uk_cv;type:varchar(64);not null" json:"component_type"`
Version string `gorm:"unique;column:uk_cv;type:varchar(64);not null" json:"version"`
StartArgs string `gorm:"column:start_args;type:varchar(1024);not null" json:"start_args"`
UpdateTime time.Time `gorm:"column:update_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"update_time"`
CreateTime time.Time `gorm:"column:create_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"create_time"`
}

// GetStartArsg pod 启动参数
func GetStartArsg(componentType, version string) (start_args string, err error) {
err = DB.Model(&TbSimulationImgCfg{}).Select("start_args").Where("component_type = ? AND version = ?", componentType,
version).
First(&start_args).Error
return
}
138 changes: 0 additions & 138 deletions dbm-services/mysql/db-simulation/model/tb_syntax_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,144 +56,6 @@ func (obj *TbSyntaxRule) GetTableName() string {
return "tb_syntax_rules"
}

// func init() {
// if err := InitRule(); err != nil {
// logger.Fatal("init syntax rule failed %s", err.Error())
// return
// }
// }

// InitRule init rules
// func InitRule() (err error) {
// initRules := []TbSyntaxRule{}
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CommandRule",
// RuleName: "HighRiskCommandRule",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(
// `["drop_table", "drop_index", "lock_tables", "drop_db", "analyze","rename_table",
// "drop_procedure", "drop_view","drop_trigger","drop_function", "drop_server",
// "drop_event", "drop_compression_dictionary","optimize", "alter_tablespace"]`),
// Desc: "高危命令",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CommandRule",
// RuleName: "BanCommandRule",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(`["truncate", "revoke", "kill", "reset", "drop_user", "grant",
// "create_user", "revoke_all", "shutdown", "lock_tables_for_backup",
// "reset", "purge", "lock_binlog_for_backup","lock_tables_for_backup",
// "install_plugin", "uninstall_plugin","alter_user"]`),
// Desc: "高危变更类型",
// WarnLevel: 1,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "SuggestBlobColumCount",
// Expr: "Val >= Item ",
// ItemType: IntItem,
// Item: []byte(`10`),
// Desc: "建议单表Blob字段不要过多",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "SuggestEngine",
// Expr: "not (Val contains Item) and ( len(Val) != 0 )",
// ItemType: StringItem,
// Item: []byte(`"innodb"`),
// Desc: "建议使用Innodb表",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "NeedPrimaryKey",
// Expr: "Val == Item",
// ItemType: IntItem,
// Item: []byte(`1`),
// Desc: "建议包含主键",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "DefinerRule",
// Expr: "Val not in Item ",
// ItemType: ArryItem,
// Item: []byte(`["ADMIN@localhost"]`),
// Desc: "必须指定definer",
// WarnLevel: 0,
// Status: true,
// })

// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "HighRiskType",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(`["drop_column"]`),
// Desc: "高危变更类型",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "HighRiskPkAlterType",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(`["add_column", "add_key", "change_column"]`),
// Desc: "主键高危变更类型",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "AlterUseAfter",
// Expr: "Val != Item",
// ItemType: StringItem,
// Item: []byte(`""`),
// Desc: "变更表时使用了after",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "AddColumnMixed",
// Expr: "( Item in Val ) && ( len(Val) > 1 )",
// ItemType: StringItem,
// Item: []byte(`"add_column"`),
// Desc: "加字段和其它alter table 类型混用，可能导致非在线加字段",
// WarnLevel: 0,
// Status: true,
// })

// initRules = append(initRules, TbSyntaxRule{
// GroupName: "DmlRule",
// RuleName: "DmlNotHasWhere",
// Expr: " Val != Item ",
// ItemType: BoolItem,
// Item: []byte(`true`),
// Desc: "没有使用WHERE或者LIMIT,可能会导致全表数据更改",
// WarnLevel: 0,
// Status: true,
// })

// for i := range initRules {
// if err = CreateRule(&initRules[i]); err != nil {
// logger.Error("初始化规则失败%s", err.Error())
// return err
// }
// }
// return err
// }

// CreateRule create rule
func CreateRule(m *TbSyntaxRule) (err error) {
return DB.Clauses(clause.OnConflict{
Expand Down

0 comments on commit d3213ee

Please sign in to comment.