feat(dbm-services): 追加部署中控tdbctl close #1721

dbm-services/mysql/db-tools/dbactuator/.gitignore

@@ -23,8 +23,7 @@ conf/
 .DS_Store
 sync_test.sh
 .vscode/
-scripts/upload_media.sh
-scripts/upload.sh
+scripts/upload*
 localtest/
 .codecc
 .idea

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/semantic_dump_schema.go

@@ -44,7 +44,7 @@ func NewSenmanticDumpSchemaCommand() *cobra.Command {
 		Use:   "semantic-dumpschema",
 		Short: "运行导出表结构",
 		Example: fmt.Sprintf(
-			`dbactuator mysql senmantic-check %s %s`,
+			`dbactuator mysql senmantic-dumpschema %s %s`,
 			subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.Service.Example()),
 		),
 		Run: func(cmd *cobra.Command, args []string) {

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/append_deploy_tdbctl.go

@@ -0,0 +1,152 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+// Package spiderctlcmd TODO
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package spiderctlcmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/internal/subcmd"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/rollback"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util"
+
+	"github.com/spf13/cobra"
+)
+
+// AppendDeployCtlSpiderAct 部署 spider ctl 实例
+type AppendDeployCtlSpiderAct struct {
+	*subcmd.BaseOptions
+	BaseService mysql.InstallMySQLComp
+}
+
+// NewAppendDeploySpiderCtlCommand godoc
+//
+// @Summary      部署 spider ctl 实例
+// @Description  部署 spider ctl 实例说明
+// @Tags         spiderctl
+// @Accept       json
+// @Param        body body      mysql.InstallMySQLComp  true  "short description"
+// @Router       /spdierctl/deploy [post]
+func NewAppendDeploySpiderCtlCommand() *cobra.Command {
+	act := AppendDeployCtlSpiderAct{
+		BaseOptions: subcmd.GBaseOptions,
+	}
+	cmd := &cobra.Command{
+		Use:   "append-deploy",
+		Short: "追加部署Spider-ctl实例",
+		Example: fmt.Sprintf(
+			`dbactuator spiderctl append-deploy %s %s`,
+			subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.BaseService.Example()),
+		),
+		Run: func(cmd *cobra.Command, args []string) {
+			util.CheckErr(act.Validate())
+			if act.RollBack {
+				util.CheckErr(act.Rollback())
+				return
+			}
+			util.CheckErr(act.Init())
+			util.CheckErr(act.Run())
+		},
+	}
+	return cmd
+}
+
+// Init 初始化
+func (d *AppendDeployCtlSpiderAct) Init() (err error) {
+	logger.Info("DeploySpiderAct Init")
+	if err = d.Deserialize(&d.BaseService.Params); err != nil {
+		logger.Error("DeserializeAndValidate failed, %v", err)
+		return err
+	}
+	d.BaseService.GeneralParam = subcmd.GeneralRuntimeParam
+
+	return d.BaseService.InitTdbctlDeploy()
+}
+
+// Rollback 回滚
+//
+//	@receiver d
+//	@return err
+func (d *AppendDeployCtlSpiderAct) Rollback() (err error) {
+	var r rollback.RollBackObjects
+	if err = d.Deserialize(&r); err != nil {
+		logger.Error("DeserializeAndValidate failed, %v", err)
+		return err
+	}
+	err = r.RollBack()
+	if err != nil {
+		logger.Error("roll back failed %s", err.Error())
+	}
+	return
+}
+
+// Run 执行
+func (d *AppendDeployCtlSpiderAct) Run() (err error) {
+	steps := subcmd.Steps{
+		{
+			FunName: "预检查",
+			Func:    d.BaseService.PreCheck,
+		},
+		{
+			FunName: "渲染my.cnf配置",
+			Func:    d.BaseService.GenerateMycnf,
+		},
+		{
+			FunName: "初始化mysqld相关目录",
+			Func:    d.BaseService.InitInstanceDirs,
+		},
+		{
+			FunName: "下载并且解压安装包",
+			Func:    d.BaseService.DecompressTdbctlPkg,
+		},
+		{
+			FunName: "初始化mysqld系统库表",
+			Func:    d.BaseService.Install,
+		},
+		{
+			FunName: "启动tdbctl",
+			Func:    d.BaseService.TdbctlStartup,
+		},
+		{
+			FunName: "执行初始化系统基础权限、库表SQL",
+			Func:    d.BaseService.InitDefaultPrivAndSchema,
+		},
+		{
+			FunName: "安装半同步复制插件",
+			Func:    d.BaseService.InstallRplSemiSyncPlugin,
+		},
+	}
+
+	if err := steps.Run(); err != nil {
+		rollbackCtxb, rerr := json.Marshal(d.BaseService.RollBackContext)
+		if rerr != nil {
+			logger.Error("json Marshal %s", err.Error())
+			fmt.Printf("<ctx>Can't RollBack<ctx>\n")
+		}
+		fmt.Printf("<ctx>%s<ctx>\n", string(rollbackCtxb))
+		return err
+	}
+
+	logger.Info("install_spider_ctl_successfully")
+	return nil
+}

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/cmd.go

@@ -1,5 +1,15 @@
-// Package spiderctlcmd 中控节点
 /*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+/*
+ * Package spiderctlcmd 中控节点操作
  * @Description: spiderctl (中控节点)相关操作的子命令集合
  */
 package spiderctlcmd
@@ -30,6 +40,8 @@ func NewSpiderCtlCommand() *cobra.Command {
 				NewClusterBackendSwitchCommand(),
 				NewClusterSchemaCheckCommand(),
 				NewClusterSchemaRepairCommand(),
+				NewAppendDeploySpiderCtlCommand(),
+				NewImportSchemaToTdbctlCommand(),
 			},
 		},
 	}

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/import_schema_from_local_spider.go

@@ -0,0 +1,81 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package spiderctlcmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/internal/subcmd"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/spiderctl"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util"
+)
+
+// ImportSchemaFromLocalSpiderAct 从本地Spider导入表结构到中控节点
+type ImportSchemaFromLocalSpiderAct struct {
+	*subcmd.BaseOptions
+	Service spiderctl.ImportSchemaFromLocalSpiderComp
+}
+
+// NewImportSchemaToTdbctlCommand create new subcommand
+func NewImportSchemaToTdbctlCommand() *cobra.Command {
+	act := ImportSchemaFromLocalSpiderAct{
+		BaseOptions: subcmd.GBaseOptions,
+	}
+	cmd := &cobra.Command{
+		Use:   "import-schema-to-tdbctl",
+		Short: "从spider节点导入表结构到中控节点",
+		Example: fmt.Sprintf(
+			`dbactuator spiderctl import-schema-to-tdbctl %s %s`,
+			subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.Service.Example()),
+		),
+		Run: func(cmd *cobra.Command, args []string) {
+			util.CheckErr(act.Validate())
+			util.CheckErr(act.Init())
+			util.CheckErr(act.Run())
+		},
+	}
+	return cmd
+}
+
+// Init 初始化
+func (d *ImportSchemaFromLocalSpiderAct) Init() (err error) {
+	logger.Info("InitCLusterRoutingAct Init")
+	if err = d.Deserialize(&d.Service.Params); err != nil {
+		logger.Error("DeserializeAndValidate failed, %v", err)
+		return err
+	}
+	d.Service.GeneralParam = subcmd.GeneralRuntimeParam
+	return
+}
+
+// Run 执行
+func (d *ImportSchemaFromLocalSpiderAct) Run() (err error) {
+	steps := subcmd.Steps{
+		{
+			FunName: "初始化",
+			Func:    d.Service.Init,
+		},
+		{
+			FunName: "从本地spider导出表结构至tdbctl",
+			Func:    d.Service.Migrate,
+		},
+	}
+
+	if err = steps.Run(); err != nil {
+		return err
+	}
+
+	logger.Info("import schema to empty tdbctl succcess ~")
+	return nil
+}

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/deploy_tdbctl.go

@@ -0,0 +1,119 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package mysql
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"regexp"
+	"strconv"
+
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/core/cst"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil"
+)
+
+// InitTdbctlDeploy TODO
+func (i *InstallMySQLComp) InitTdbctlDeploy() (err error) {
+	i.WorkUser = "root"
+	i.WorkPassword = ""
+	i.InstallDir = cst.UsrLocal
+	i.MysqlInstallDir = cst.MysqldInstallPath
+	i.TdbctlInstallDir = cst.TdbctlInstallPath
+	i.DataRootPath = cst.DefaultMysqlDataRootPath
+	i.LogRootPath = cst.DefaultMysqlLogRootPath
+	i.DefaultMysqlDataDirName = cst.DefaultMysqlDataBasePath
+	i.DefaultMysqlLogDirName = cst.DefaultMysqlLogBasePath
+	// 计算获取需要安装的ports
+	i.InsPorts = i.Params.Ports
+	i.MyCnfTpls = make(map[int]*util.CnfFile)
+	var mountpoint string
+	if mountpoint, err = osutil.FindFirstMountPointProxy(
+		cst.DefaultProxyDataRootPath,
+		cst.AlterNativeProxyDataRootPath,
+	); err != nil {
+		logger.Error("not found mount point /data1")
+		return err
+	}
+	i.DataRootPath = mountpoint
+	i.DataBaseDir = path.Join(mountpoint, cst.DefaultMysqlDataBasePath)
+	i.LogRootPath = mountpoint
+	i.LogBaseDir = path.Join(mountpoint, cst.DefaultMysqlLogBasePath)
+
+	// 反序列化mycnf 配置
+	var mycnfs map[Port]json.RawMessage
+	if err = json.Unmarshal([]byte(i.Params.MyCnfConfigs), &mycnfs); err != nil {
+		logger.Error("反序列化配置失败:%s", err.Error())
+		return err
+	}
+	for _, port := range i.InsPorts {
+		var cnfraw json.RawMessage
+		var ok bool
+		if cnfraw, ok = mycnfs[port]; !ok {
+			return fmt.Errorf("参数中没有%d的配置", port)
+		}
+		var mycnf mysqlutil.MycnfObject
+		if err = json.Unmarshal(cnfraw, &mycnf); err != nil {
+			logger.Error("反序列%d 化配置失败:%s", port, err.Error())
+			return err
+		}
+		cnftpl, err := util.NewMyCnfObject(mycnf, "tpl")
+		if err != nil {
+			logger.Error("初始化mycnf ini 模版:%s", err.Error())
+			return err
+		}
+		//  删除 innodb 相关配置参数
+		innodbRe := regexp.MustCompile("^innodb")
+		for _, item := range cnftpl.Cfg.Section(util.MysqldSec).Keys() {
+			if innodbRe.MatchString(item.Name()) {
+				cnftpl.Cfg.Section(util.MysqldSec).DeleteKey(item.Name())
+			}
+		}
+		i.MyCnfTpls[port] = cnftpl
+	}
+	// 计算需要替换的参数配置
+	if err := i.replacecnf(); err != nil {
+		return err
+	}
+	i.Checkfunc = append(i.Checkfunc, i.CheckTimeZoneSetting)
+	i.Checkfunc = append(i.Checkfunc, i.precheckMysqlPackageBitOS)
+	i.Checkfunc = append(i.Checkfunc, i.Params.Medium.Check)
+	return nil
+}
+
+func (i *InstallMySQLComp) replacecnf() error {
+	i.RenderConfigs = make(map[int]RenderConfigs)
+	i.InsInitDirs = make(map[int]InitDirs)
+	i.InsSockets = make(map[int]string)
+	for _, port := range i.InsPorts {
+		insBaseDataDir := path.Join(i.DataBaseDir, strconv.Itoa(port))
+		insBaseLogDir := path.Join(i.LogBaseDir, strconv.Itoa(port))
+		serverId, err := mysqlutil.GenMysqlServerId(i.Params.Host, port)
+		if err != nil {
+			logger.Error("%s:%d generation serverId Failed %s", i.Params.Host, port, err.Error())
+			return err
+		}
+		i.RenderConfigs[port] = RenderConfigs{Mysqld{
+			Datadir:            insBaseDataDir,
+			Logdir:             insBaseLogDir,
+			ServerId:           serverId,
+			Port:               strconv.Itoa(port),
+			CharacterSetServer: i.Params.CharSet,
+			BindAddress:        i.Params.Host,
+		}}
+
+		i.InsInitDirs[port] = append(i.InsInitDirs[port], []string{insBaseDataDir, insBaseLogDir}...)
+	}
+	return nil
+}