feat(dbm-services): 模拟执行时，复制源实例的关键启动参数 TencentBlueKing#5759

dbm-services/mysql/db-simulation/app/service/kubernets.go

@@ -56,6 +56,7 @@ type KubeClientSets struct {
 type MySQLPodBaseInfo struct {
 	PodName string
 	Lables  map[string]string
+	Args    []string
 	RootPwd string
 	Charset string
 }
@@ -146,7 +147,7 @@ func (k *DbPodSets) CreateClusterPod() (err error) {
 					Resources:       k.getResourceLimit(),
 					ImagePullPolicy: v1.PullIfNotPresent,
 					Image:           k.DbImage,
-					Args: []string{"mysqld", "--defaults-file=/etc/my.cnf", "--log_bin_trust_function_creators", "--port=20000",
+					Args: []string{"--defaults-file=/etc/my.cnf", "--log_bin_trust_function_creators", "--port=20000",
 						fmt.Sprintf("--character-set-server=%s",
 							k.BaseInfo.Charset),
 						"--user=mysql"},
@@ -168,7 +169,7 @@ func (k *DbPodSets) CreateClusterPod() (err error) {
 					Resources:       k.getResourceLimit(),
 					ImagePullPolicy: v1.PullIfNotPresent,
 					Image:           k.SpiderImage,
-					Args: []string{"mysqld", "--defaults-file=/etc/my.cnf", "--log_bin_trust_function_creators", "--port=25000",
+					Args: []string{"--defaults-file=/etc/my.cnf", "--log_bin_trust_function_creators", "--port=25000",
 						fmt.Sprintf("--character-set-server=%s",
 							k.BaseInfo.Charset),
 						"--user=mysql"},
@@ -191,7 +192,7 @@ func (k *DbPodSets) CreateClusterPod() (err error) {
 					Resources:       k.gettdbctlResourceLimit(),
 					ImagePullPolicy: v1.PullIfNotPresent,
 					Image:           k.TdbCtlImage,
-					Args: []string{"mysqld", "--defaults-file=/etc/my.cnf", "--port=26000", "--tc-admin=1",
+					Args: []string{"--defaults-file=/etc/my.cnf", "--port=26000", "--tc-admin=1",
 						"--dbm-allow-standalone-primary",
 						fmt.Sprintf("--character-set-server=%s",
 							k.BaseInfo.Charset),
@@ -326,6 +327,11 @@ func (k *DbPodSets) gettdbctlResourceLimit() v1.ResourceRequirements {
 
 // CreateMySQLPod create mysql pod
 func (k *DbPodSets) CreateMySQLPod() (err error) {
+	startArgs := []string{"--defaults-file=/etc/my.cnf", "--skip-log-bin",
+		fmt.Sprintf("--character-set-server=%s", k.BaseInfo.Charset)}
+	startArgs = append(startArgs, k.BaseInfo.Args...)
+	startArgs = append(startArgs, "--user=mysql")
+	logger.Info("start pod args %v", startArgs)
 	c := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Pod",
@@ -354,10 +360,7 @@ func (k *DbPodSets) CreateMySQLPod() (err error) {
 				},
 				ImagePullPolicy: v1.PullIfNotPresent,
 				Image:           k.DbImage,
-				Args: []string{"mysqld", "--defaults-file=/etc/my.cnf", "--log-bin-trust-function-creators", "--skip-log-bin",
-					fmt.Sprintf("--character-set-server=%s",
-						k.BaseInfo.Charset),
-					"--user=mysql"},
+				Args:            startArgs,
 				ReadinessProbe: &v1.Probe{
 					ProbeHandler: v1.ProbeHandler{
 						Exec: &v1.ExecAction{

dbm-services/mysql/db-simulation/app/service/simulation_task.go

@@ -33,16 +33,35 @@ var DelPod bool = true
 
 // BaseParam 请求模拟执行的基础参数
 type BaseParam struct {
-	Uid           string             `json:"uid"`
-	NodeId        string             `json:"node_id"`
-	RootId        string             `json:"root_id"`
-	VersionId     string             `json:"version_id"`
-	TaskId        string             `json:"task_id"  binding:"required"`
-	MySQLVersion  string             `json:"mysql_version"  binding:"required"`
-	MySQLCharSet  string             `json:"mysql_charset"  binding:"required"`
-	Path          string             `json:"path"  binding:"required"`
-	SchemaSQLFile string             `json:"schema_sql_file"  binding:"required"`
-	ExcuteObjects []ExcuteSQLFileObj `json:"execute_objects"  binding:"gt=0,dive,required"`
+	//nolint
+	Uid string `json:"uid"`
+	//nolint
+	NodeId string `json:"node_id"`
+	//nolint
+	RootId string `json:"root_id"`
+	//nolint
+	VersionId string `json:"version_id"`
+	//nolint
+	TaskId            string             `json:"task_id"  binding:"required"`
+	MySQLVersion      string             `json:"mysql_version"  binding:"required"`
+	MySQLCharSet      string             `json:"mysql_charset"  binding:"required"`
+	MySQLStartConfigs map[string]string  `json:"mysql_start_config"`
+	Path              string             `json:"path"  binding:"required"`
+	SchemaSQLFile     string             `json:"schema_sql_file"  binding:"required"`
+	ExcuteObjects     []ExcuteSQLFileObj `json:"execute_objects"  binding:"gt=0,dive,required"`
+}
+
+// BuildStartArgs mysql pod start args
+func (b BaseParam) BuildStartArgs() []string {
+	if len(b.MySQLStartConfigs) == 0 {
+		return []string{}
+	}
+	var args []string
+	for key, val := range b.MySQLStartConfigs {
+		p := strings.ReplaceAll(strings.TrimSpace(key), "_", "-")
+		args = append(args, fmt.Sprintf("--%s=%s", p, strings.TrimSpace(val)))
+	}
+	return args
 }
 
 // SpiderSimulationExecParam TODO

dbm-services/mysql/db-simulation/handler/handler.go

@@ -103,6 +103,7 @@ func SpiderClusterSimulation(r *gin.Context) {
 		Lables: map[string]string{"task_id": replaceUnderSource(param.TaskId),
 			"request_id": requestId},
 		RootPwd: rootPwd,
+		Args:    param.BuildStartArgs(),
 		Charset: param.MySQLCharSet,
 	}
 	service.SpiderTaskChan <- tsk
@@ -145,6 +146,7 @@ func Dbsimulation(r *gin.Context) {
 		Lables: map[string]string{"task_id": replaceUnderSource(param.TaskId),
 			"request_id": requestId},
 		RootPwd: param.TaskId,
+		Args:    param.BuildStartArgs(),
 		Charset: param.MySQLCharSet,
 	}
 	service.TaskChan <- tsk

dbm-ui/backend/flow/engine/bamboo/scene/mysql/common/common_sub_flow.py

@@ -16,10 +16,8 @@
 
 from backend import env
 from backend.components import DBConfigApi
-from backend.components.db_remote_service.client import DRSApi
 from backend.components.dbconfig.constants import FormatType, LevelName
 from backend.configuration.constants import DBType
-from backend.constants import IP_PORT_DIVIDER
 from backend.db_meta.enums import ClusterType, InstanceInnerRole
 from backend.db_meta.models import Cluster, StorageInstance
 from backend.flow.consts import (
@@ -57,6 +55,7 @@
     YumInstallPerlKwargs,
 )
 from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload
+from backend.flow.utils.mysql.mysql_commom_query import query_mysql_variables
 from backend.flow.utils.mysql.mysql_db_meta import MySQLDBMeta
 
 logger = logging.getLogger("flow")
@@ -817,34 +816,6 @@ def update_machine_system_info_flow(
     return sub_pipeline.build_sub_process(sub_name=_("获取机器系统信息"))
 
 
-def query_mysql_variables(host: str, port: int, bk_cloud_id: int):
-    """
-    查询远程节点变量
-    """
-    body = {
-        "addresses": ["{}{}{}".format(host, IP_PORT_DIVIDER, port)],
-        "cmds": ["show global variables;"],
-        "force": False,
-        "bk_cloud_id": bk_cloud_id,
-    }
-    resp = DRSApi.rpc(body)
-    logger.info(f"query charset {resp}")
-    if not resp and len(resp) < 1:
-        raise Exception(_("DRS{}:{}查询变量失败,返回为空值").format(host, port))
-
-    if not resp[0]["cmd_results"]:
-        raise Exception(_("DRS查询字符集失败：{}").format(resp[0]["error_msg"]))
-
-    var_list = resp[0]["cmd_results"][0]["table_data"]
-
-    var_map = {}
-    for var_item in var_list:
-        var_name = var_item["Variable_name"]
-        val = var_item["Value"]
-        var_map[var_name] = val
-    return var_map
-
-
 def sync_mycnf_item_sub_flow(
     uid: str,
     root_id: str,

dbm-ui/backend/flow/engine/bamboo/scene/mysql/import_sqlfile_flow.py

@@ -16,10 +16,8 @@
 from django.utils.translation import ugettext as _
 
 from backend.components import DBConfigApi
-from backend.components.db_remote_service.client import DRSApi
 from backend.components.dbconfig.constants import FormatType, LevelName
 from backend.configuration.constants import DBType
-from backend.constants import IP_PORT_DIVIDER
 from backend.core import consts
 from backend.db_meta.enums.instance_role import InstanceRole
 from backend.db_meta.models import Cluster, StorageInstance
@@ -33,6 +31,7 @@
 from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent
 from backend.flow.utils.mysql.mysql_act_dataclass import DownloadMediaKwargs, ExecActuatorKwargs
 from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload
+from backend.flow.utils.mysql.mysql_commom_query import query_mysql_variables
 from backend.ticket.constants import TicketType
 
 logger = logging.getLogger("flow")
@@ -177,8 +176,12 @@ def sql_semantic_check_flow(self):
             backend_ip = template_cluster["backend_ip"]
             backend_port = template_cluster["port"]
             bk_cloud_id = template_cluster["bk_cloud_id"]
-            backend_charset = self.__get_backend_charset(backend_ip, backend_port, bk_cloud_id)
-            logger.info(f"backend_charset: {backend_charset}")
+            origin_mysql_var_map = query_mysql_variables(host=backend_ip, port=backend_port, bk_cloud_id=bk_cloud_id)
+            backend_charset = origin_mysql_var_map.get("character_set_client")
+            start_mysqld_configs = {}
+            for var in ["sql_mode", "lower_case_table_names", "log_bin_trust_function_creators"]:
+                if origin_mysql_var_map.__contains__(var):
+                    start_mysqld_configs[var] = origin_mysql_var_map.get(var)
 
             sub_pipeline.add_act(
                 act_name=_("给模板集群下发db-actuator"),
@@ -218,6 +221,7 @@ def sql_semantic_check_flow(self):
                         "task_id": self.root_id,
                         "schema_sql_file": self.semantic_dump_schema_file_name,
                         "execute_objects": self.data["execute_objects"],
+                        "mysql_start_config": start_mysqld_configs,
                     },
                 },
             )
@@ -286,25 +290,3 @@ def __get_version_and_charset(self, db_module_id, cluster_type) -> Any:
             }
         )["content"]
         return data["db_version"]
-
-    def __get_backend_charset(self, ip, port, bk_cloud_id) -> str:
-        # 获取远端字符集
-        logger.info(f"param: {ip}:{port}")
-        body = {
-            "addresses": ["{}{}{}".format(ip, IP_PORT_DIVIDER, port)],
-            "cmds": ["show global variables like 'character_set_client'"],
-            "force": False,
-            "bk_cloud_id": bk_cloud_id,
-        }
-
-        resp = DRSApi.rpc(body)
-        logger.info(f"query charset {resp}")
-
-        if not resp[0]["cmd_results"]:
-            raise Exception(_("DRS查询字符集失败：{}").format(resp[0]["error_msg"]))
-
-        charset = resp[0]["cmd_results"][0]["table_data"][0]["Value"]
-        if not charset:
-            logger.error(_("获取字符集为空..."))
-            raise Exception(_("获取字符集为空"))
-        return charset

dbm-ui/backend/flow/utils/mysql/mysql_commom_query.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging.config
+
+from django.utils.translation import gettext as _
+
+from backend.components.db_remote_service.client import DRSApi
+from backend.constants import IP_PORT_DIVIDER
+
+logger = logging.getLogger("flow")
+
+
+def query_mysql_variables(host: str, port: int, bk_cloud_id: int):
+    """
+    查询远程节点变量
+    """
+    body = {
+        "addresses": ["{}{}{}".format(host, IP_PORT_DIVIDER, port)],
+        "cmds": ["show global variables;"],
+        "force": False,
+        "bk_cloud_id": bk_cloud_id,
+    }
+    resp = DRSApi.rpc(body)
+    logger.info(f"query vaiables {resp}")
+    if not resp and len(resp) < 1:
+        raise Exception(_("DRS{}:{}查询变量失败,返回为空值").format(host, port))
+
+    if not resp[0]["cmd_results"]:
+        raise Exception(_("DRS查询字符集失败：{}").format(resp[0]["error_msg"]))
+
+    var_list = resp[0]["cmd_results"][0]["table_data"]
+
+    var_map = {}
+    for var_item in var_list:
+        var_name = var_item["Variable_name"]
+        val = var_item["Value"]
+        var_map[var_name] = val
+    return var_map