From 53cdf8699bb9b54340249ca8a841b8bcfc0efc05 Mon Sep 17 00:00:00 2001 From: ymakedaq <996156275@qq.com> Date: Tue, 14 Nov 2023 11:18:34 +0800 Subject: [PATCH] =?UTF-8?q?feat(dbm-services):=20=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E4=B8=AD=E6=8E=A7tdbctl=20close=20#1721?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/db-tools/dbactuator/.gitignore | 3 +- .../subcmd/mysqlcmd/semantic_dump_schema.go | 2 +- .../spiderctlcmd/append_deploy_tdbctl.go | 152 ++++++++++++++ .../internal/subcmd/spiderctlcmd/cmd.go | 14 +- .../import_schema_from_local_spider.go | 81 ++++++++ .../pkg/components/mysql/deploy_tdbctl.go | 119 +++++++++++ .../pkg/components/mysql/install_mysql.go | 22 ++- .../import_schema_from_local_spider.go | 185 ++++++++++++++++++ dbm-ui/backend/flow/consts.py | 1 + .../bamboo/scene/common/get_file_list.py | 9 + .../spider/migrate_spider_cluster_from_gcs.py | 171 ++++++++++++++++ .../backend/flow/engine/controller/spider.py | 8 + dbm-ui/backend/flow/urls.py | 3 + .../flow/utils/mysql/mysql_act_playload.py | 34 ++++ .../views/migrate_spider_cluster_from_gcs.py | 37 ++++ 15 files changed, 831 insertions(+), 10 deletions(-) create mode 100644 dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/append_deploy_tdbctl.go create mode 100644 dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/import_schema_from_local_spider.go create mode 100644 dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/deploy_tdbctl.go create mode 100644 dbm-services/mysql/db-tools/dbactuator/pkg/components/spiderctl/import_schema_from_local_spider.go create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/spider/migrate_spider_cluster_from_gcs.py create mode 100644 dbm-ui/backend/flow/views/migrate_spider_cluster_from_gcs.py diff --git a/dbm-services/mysql/db-tools/dbactuator/.gitignore b/dbm-services/mysql/db-tools/dbactuator/.gitignore index 3810e2c293..eda3e9f9f1 100644 --- a/dbm-services/mysql/db-tools/dbactuator/.gitignore +++ b/dbm-services/mysql/db-tools/dbactuator/.gitignore @@ -23,8 +23,7 @@ conf/ .DS_Store sync_test.sh .vscode/ -scripts/upload_media.sh -scripts/upload.sh +scripts/upload* localtest/ .codecc .idea diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/semantic_dump_schema.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/semantic_dump_schema.go index cc7e6d8143..7e7beed46f 100644 --- a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/semantic_dump_schema.go +++ b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/semantic_dump_schema.go @@ -44,7 +44,7 @@ func NewSenmanticDumpSchemaCommand() *cobra.Command { Use: "semantic-dumpschema", Short: "运行导出表结构", Example: fmt.Sprintf( - `dbactuator mysql senmantic-check %s %s`, + `dbactuator mysql senmantic-dumpschema %s %s`, subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.Service.Example()), ), Run: func(cmd *cobra.Command, args []string) { diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/append_deploy_tdbctl.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/append_deploy_tdbctl.go new file mode 100644 index 0000000000..2b142f4470 --- /dev/null +++ b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/append_deploy_tdbctl.go @@ -0,0 +1,152 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +// Package spiderctlcmd TODO +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package spiderctlcmd + +import ( + "encoding/json" + "fmt" + + "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-tools/dbactuator/internal/subcmd" + "dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql" + "dbm-services/mysql/db-tools/dbactuator/pkg/rollback" + "dbm-services/mysql/db-tools/dbactuator/pkg/util" + + "github.com/spf13/cobra" +) + +// AppendDeployCtlSpiderAct 部署 spider ctl 实例 +type AppendDeployCtlSpiderAct struct { + *subcmd.BaseOptions + BaseService mysql.InstallMySQLComp +} + +// NewAppendDeploySpiderCtlCommand godoc +// +// @Summary 部署 spider ctl 实例 +// @Description 部署 spider ctl 实例说明 +// @Tags spiderctl +// @Accept json +// @Param body body mysql.InstallMySQLComp true "short description" +// @Router /spdierctl/deploy [post] +func NewAppendDeploySpiderCtlCommand() *cobra.Command { + act := AppendDeployCtlSpiderAct{ + BaseOptions: subcmd.GBaseOptions, + } + cmd := &cobra.Command{ + Use: "append-deploy", + Short: "追加部署Spider-ctl实例", + Example: fmt.Sprintf( + `dbactuator spiderctl append-deploy %s %s`, + subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.BaseService.Example()), + ), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(act.Validate()) + if act.RollBack { + util.CheckErr(act.Rollback()) + return + } + util.CheckErr(act.Init()) + util.CheckErr(act.Run()) + }, + } + return cmd +} + +// Init 初始化 +func (d *AppendDeployCtlSpiderAct) Init() (err error) { + logger.Info("DeploySpiderAct Init") + if err = d.Deserialize(&d.BaseService.Params); err != nil { + logger.Error("DeserializeAndValidate failed, %v", err) + return err + } + d.BaseService.GeneralParam = subcmd.GeneralRuntimeParam + + return d.BaseService.InitTdbctlDeploy() +} + +// Rollback 回滚 +// +// @receiver d +// @return err +func (d *AppendDeployCtlSpiderAct) Rollback() (err error) { + var r rollback.RollBackObjects + if err = d.Deserialize(&r); err != nil { + logger.Error("DeserializeAndValidate failed, %v", err) + return err + } + err = r.RollBack() + if err != nil { + logger.Error("roll back failed %s", err.Error()) + } + return +} + +// Run 执行 +func (d *AppendDeployCtlSpiderAct) Run() (err error) { + steps := subcmd.Steps{ + { + FunName: "预检查", + Func: d.BaseService.PreCheck, + }, + { + FunName: "渲染my.cnf配置", + Func: d.BaseService.GenerateMycnf, + }, + { + FunName: "初始化mysqld相关目录", + Func: d.BaseService.InitInstanceDirs, + }, + { + FunName: "下载并且解压安装包", + Func: d.BaseService.DecompressTdbctlPkg, + }, + { + FunName: "初始化mysqld系统库表", + Func: d.BaseService.Install, + }, + { + FunName: "启动tdbctl", + Func: d.BaseService.TdbctlStartup, + }, + { + FunName: "执行初始化系统基础权限、库表SQL", + Func: d.BaseService.InitDefaultPrivAndSchema, + }, + { + FunName: "安装半同步复制插件", + Func: d.BaseService.InstallRplSemiSyncPlugin, + }, + } + + if err := steps.Run(); err != nil { + rollbackCtxb, rerr := json.Marshal(d.BaseService.RollBackContext) + if rerr != nil { + logger.Error("json Marshal %s", err.Error()) + fmt.Printf("Can't RollBack\n") + } + fmt.Printf("%s\n", string(rollbackCtxb)) + return err + } + + logger.Info("install_spider_ctl_successfully") + return nil +} diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/cmd.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/cmd.go index e764e29724..2a62134d10 100644 --- a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/cmd.go +++ b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/cmd.go @@ -1,5 +1,15 @@ -// Package spiderctlcmd 中控节点 /* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Package spiderctlcmd 中控节点操作 * @Description: spiderctl (中控节点)相关操作的子命令集合 */ package spiderctlcmd @@ -30,6 +40,8 @@ func NewSpiderCtlCommand() *cobra.Command { NewClusterBackendSwitchCommand(), NewClusterSchemaCheckCommand(), NewClusterSchemaRepairCommand(), + NewAppendDeploySpiderCtlCommand(), + NewImportSchemaToTdbctlCommand(), }, }, } diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/import_schema_from_local_spider.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/import_schema_from_local_spider.go new file mode 100644 index 0000000000..1f2c36fd0d --- /dev/null +++ b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/spiderctlcmd/import_schema_from_local_spider.go @@ -0,0 +1,81 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package spiderctlcmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-tools/dbactuator/internal/subcmd" + "dbm-services/mysql/db-tools/dbactuator/pkg/components/spiderctl" + "dbm-services/mysql/db-tools/dbactuator/pkg/util" +) + +// ImportSchemaFromLocalSpiderAct 从本地Spider导入表结构到中控节点 +type ImportSchemaFromLocalSpiderAct struct { + *subcmd.BaseOptions + Service spiderctl.ImportSchemaFromLocalSpiderComp +} + +// NewImportSchemaToTdbctlCommand create new subcommand +func NewImportSchemaToTdbctlCommand() *cobra.Command { + act := ImportSchemaFromLocalSpiderAct{ + BaseOptions: subcmd.GBaseOptions, + } + cmd := &cobra.Command{ + Use: "import-schema-to-tdbctl", + Short: "从spider节点导入表结构到中控节点", + Example: fmt.Sprintf( + `dbactuator spiderctl import-schema-to-tdbctl %s %s`, + subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.Service.Example()), + ), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(act.Validate()) + util.CheckErr(act.Init()) + util.CheckErr(act.Run()) + }, + } + return cmd +} + +// Init 初始化 +func (d *ImportSchemaFromLocalSpiderAct) Init() (err error) { + logger.Info("InitCLusterRoutingAct Init") + if err = d.Deserialize(&d.Service.Params); err != nil { + logger.Error("DeserializeAndValidate failed, %v", err) + return err + } + d.Service.GeneralParam = subcmd.GeneralRuntimeParam + return +} + +// Run 执行 +func (d *ImportSchemaFromLocalSpiderAct) Run() (err error) { + steps := subcmd.Steps{ + { + FunName: "初始化", + Func: d.Service.Init, + }, + { + FunName: "从本地spider导出表结构至tdbctl", + Func: d.Service.Migrate, + }, + } + + if err = steps.Run(); err != nil { + return err + } + + logger.Info("import schema to empty tdbctl succcess ~") + return nil +} diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/deploy_tdbctl.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/deploy_tdbctl.go new file mode 100644 index 0000000000..bdcd06d890 --- /dev/null +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/deploy_tdbctl.go @@ -0,0 +1,119 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package mysql + +import ( + "encoding/json" + "fmt" + "path" + "regexp" + "strconv" + + "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" + "dbm-services/mysql/db-tools/dbactuator/pkg/util" + "dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil" + "dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil" +) + +// InitTdbctlDeploy TODO +func (i *InstallMySQLComp) InitTdbctlDeploy() (err error) { + i.WorkUser = "root" + i.WorkPassword = "" + i.InstallDir = cst.UsrLocal + i.MysqlInstallDir = cst.MysqldInstallPath + i.TdbctlInstallDir = cst.TdbctlInstallPath + i.DataRootPath = cst.DefaultMysqlDataRootPath + i.LogRootPath = cst.DefaultMysqlLogRootPath + i.DefaultMysqlDataDirName = cst.DefaultMysqlDataBasePath + i.DefaultMysqlLogDirName = cst.DefaultMysqlLogBasePath + // 计算获取需要安装的ports + i.InsPorts = i.Params.Ports + i.MyCnfTpls = make(map[int]*util.CnfFile) + var mountpoint string + if mountpoint, err = osutil.FindFirstMountPointProxy( + cst.DefaultProxyDataRootPath, + cst.AlterNativeProxyDataRootPath, + ); err != nil { + logger.Error("not found mount point /data1") + return err + } + i.DataRootPath = mountpoint + i.DataBaseDir = path.Join(mountpoint, cst.DefaultMysqlDataBasePath) + i.LogRootPath = mountpoint + i.LogBaseDir = path.Join(mountpoint, cst.DefaultMysqlLogBasePath) + + // 反序列化mycnf 配置 + var mycnfs map[Port]json.RawMessage + if err = json.Unmarshal([]byte(i.Params.MyCnfConfigs), &mycnfs); err != nil { + logger.Error("反序列化配置失败:%s", err.Error()) + return err + } + for _, port := range i.InsPorts { + var cnfraw json.RawMessage + var ok bool + if cnfraw, ok = mycnfs[port]; !ok { + return fmt.Errorf("参数中没有%d的配置", port) + } + var mycnf mysqlutil.MycnfObject + if err = json.Unmarshal(cnfraw, &mycnf); err != nil { + logger.Error("反序列%d 化配置失败:%s", port, err.Error()) + return err + } + cnftpl, err := util.NewMyCnfObject(mycnf, "tpl") + if err != nil { + logger.Error("初始化mycnf ini 模版:%s", err.Error()) + return err + } + // 删除 innodb 相关配置参数 + innodbRe := regexp.MustCompile("^innodb") + for _, item := range cnftpl.Cfg.Section(util.MysqldSec).Keys() { + if innodbRe.MatchString(item.Name()) { + cnftpl.Cfg.Section(util.MysqldSec).DeleteKey(item.Name()) + } + } + i.MyCnfTpls[port] = cnftpl + } + // 计算需要替换的参数配置 + if err := i.replacecnf(); err != nil { + return err + } + i.Checkfunc = append(i.Checkfunc, i.CheckTimeZoneSetting) + i.Checkfunc = append(i.Checkfunc, i.precheckMysqlPackageBitOS) + i.Checkfunc = append(i.Checkfunc, i.Params.Medium.Check) + return nil +} + +func (i *InstallMySQLComp) replacecnf() error { + i.RenderConfigs = make(map[int]RenderConfigs) + i.InsInitDirs = make(map[int]InitDirs) + i.InsSockets = make(map[int]string) + for _, port := range i.InsPorts { + insBaseDataDir := path.Join(i.DataBaseDir, strconv.Itoa(port)) + insBaseLogDir := path.Join(i.LogBaseDir, strconv.Itoa(port)) + serverId, err := mysqlutil.GenMysqlServerId(i.Params.Host, port) + if err != nil { + logger.Error("%s:%d generation serverId Failed %s", i.Params.Host, port, err.Error()) + return err + } + i.RenderConfigs[port] = RenderConfigs{Mysqld{ + Datadir: insBaseDataDir, + Logdir: insBaseLogDir, + ServerId: serverId, + Port: strconv.Itoa(port), + CharacterSetServer: i.Params.CharSet, + BindAddress: i.Params.Host, + }} + + i.InsInitDirs[port] = append(i.InsInitDirs[port], []string{insBaseDataDir, insBaseLogDir}...) + } + return nil +} diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go index 040be01499..d600e744b5 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go @@ -1,3 +1,13 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + package mysql import ( @@ -599,8 +609,8 @@ func (i *InstallMySQLComp) Startup() (err error) { s := computil.StartMySQLParam{ MediaDir: i.MysqlInstallDir, MyCnfName: util.GetMyCnfFileName(port), - MySQLUser: i.WorkUser, //"root", - MySQLPwd: i.WorkPassword, //"", + MySQLUser: i.WorkUser, // "root", + MySQLPwd: i.WorkPassword, // "", Socket: i.InsSockets[port], SkipSlaveFlag: false, } @@ -936,8 +946,8 @@ func (i *InstallMySQLComp) TdbctlStartup() (err error) { s := computil.StartMySQLParam{ MediaDir: i.TdbctlInstallDir, MyCnfName: util.GetMyCnfFileName(port), - MySQLUser: i.WorkUser, //"root", - MySQLPwd: i.WorkPassword, //"", + MySQLUser: i.WorkUser, // "root", + MySQLPwd: i.WorkPassword, // "", Socket: i.InsSockets[port], SkipSlaveFlag: false, } @@ -1018,8 +1028,8 @@ func (i *InstallMySQLComp) getSuperUserAccountForSpider() (initAccountsql []stri func (i *InstallMySQLComp) create_spider_table(socket string) (err error) { return mysqlutil.ExecuteSqlAtLocal{ - User: i.WorkUser, //"root", - Password: i.WorkPassword, //"", + User: i.WorkUser, // "root", + Password: i.WorkPassword, // "", Socket: socket, Charset: i.Params.CharSet, }.ExcuteSqlByMySQLClientOne(path.Join(i.MysqlInstallDir, "scripts/install_spider.sql"), "") diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/spiderctl/import_schema_from_local_spider.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/spiderctl/import_schema_from_local_spider.go new file mode 100644 index 0000000000..2addcaae9c --- /dev/null +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/spiderctl/import_schema_from_local_spider.go @@ -0,0 +1,185 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package spiderctl + +import ( + "path" + "regexp" + "time" + + "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-tools/dbactuator/pkg/components" + "dbm-services/mysql/db-tools/dbactuator/pkg/components/computil" + "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" + "dbm-services/mysql/db-tools/dbactuator/pkg/native" + "dbm-services/mysql/db-tools/dbactuator/pkg/util" + "dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil" +) + +// ImportSchemaFromLocalSpiderComp TODO +type ImportSchemaFromLocalSpiderComp struct { + GeneralParam *components.GeneralParam `json:"general"` + Params ImportSchemaFromLocalSpiderParam `json:"extend"` + importSchemaFromLocalSpiderRuntime +} + +// ImportSchemaFromLocalSpiderParam TODO +type ImportSchemaFromLocalSpiderParam struct { + Host string `json:"host" validate:"required,ip"` // 当前实例的主机地址 + Port int `json:"port" validate:"required,lt=65536,gte=3306"` // 当前实例的端口 + SpiderPort int `json:"spider_port" validate:"required,lt=65536,gte=3306"` // spider节点端口 +} + +type importSchemaFromLocalSpiderRuntime struct { + spiderconn *native.DbWorker + tdbctlConn *native.DbWorker + version string + charset string + dumpDbs []string + tmpDumpDir string + tmpDumpFile string + tdbctlSocket string +} + +// Example subcommand example input +func (i *ImportSchemaFromLocalSpiderComp) Example() interface{} { + comp := ImportSchemaFromLocalSpiderComp{ + Params: ImportSchemaFromLocalSpiderParam{ + Host: "1.1.1.1", + Port: 26000, + SpiderPort: 25000, + }, + } + return comp +} + +// Init prepare run env +func (c *ImportSchemaFromLocalSpiderComp) Init() (err error) { + conn, err := native.InsObject{ + Host: c.Params.Host, + Port: c.Params.SpiderPort, + User: c.GeneralParam.RuntimeAccountParam.AdminUser, + Pwd: c.GeneralParam.RuntimeAccountParam.AdminPwd, + }.Conn() + if err != nil { + logger.Error("Connect spider %d failed:%s", c.Params.Port, err.Error()) + return err + } + c.tdbctlConn, err = native.InsObject{ + Host: c.Params.Host, + Port: c.Params.SpiderPort, + User: c.GeneralParam.RuntimeAccountParam.AdminUser, + Pwd: c.GeneralParam.RuntimeAccountParam.AdminPwd, + }.Conn() + if err != nil { + logger.Error("Connect tdbctl %d failed:%s", c.Params.Port, err.Error()) + return err + } + c.tdbctlSocket, err = c.tdbctlConn.ShowSocket() + if err != nil { + logger.Warn("get tdbctl socket failed %s", err.Error()) + err = nil + } + alldbs, err := conn.ShowDatabases() + if err != nil { + logger.Error("show all databases failed:%s", err.Error()) + return err + } + version, err := conn.SelectVersion() + if err != nil { + logger.Error("获取version failed %s", err.Error()) + return err + } + finaldbs := []string{} + reg := regexp.MustCompile(`^bak_cbs`) + for _, db := range util.FilterOutStringSlice(alldbs, computil.GetGcsSystemDatabasesIgnoreTest(version)) { + if reg.MatchString(db) { + continue + } + finaldbs = append(finaldbs, db) + } + c.spiderconn = conn + c.dumpDbs = finaldbs + c.tmpDumpDir = path.Join(cst.BK_PKG_INSTALL_PATH, "schema_migrate") + c.tmpDumpFile = time.Now().Format(cst.TimeLayoutDir) + "_schema.sql" + return err +} + +// Migrate TODO +func (c *ImportSchemaFromLocalSpiderComp) Migrate() (err error) { + if len(c.dumpDbs) == 0 { + logger.Info("当前没有需要拷贝的表，请检查，直接返回") + return nil + } + if err = c.dumpSchema(); err != nil { + logger.Error("dump schema failed %s", err.Error()) + return err + } + if err = c.loadSchema(); err != nil { + logger.Error("load schema failed %s", err.Error()) + return err + } + return nil +} + +// dumpSchema 运行备份表结构 +// +// @receiver c +// @return err +func (c *ImportSchemaFromLocalSpiderComp) dumpSchema() (err error) { + var dumper mysqlutil.Dumper + dumpOption := mysqlutil.MySQLDumpOption{ + NoData: true, + AddDropTable: true, + NeedUseDb: true, + DumpRoutine: true, + DumpTrigger: true, + DumpEvent: true, + } + + dumper = &mysqlutil.MySQLDumperTogether{ + MySQLDumper: mysqlutil.MySQLDumper{ + DumpDir: c.tmpDumpDir, + Ip: c.Params.Host, + Port: c.Params.SpiderPort, + DbBackupUser: c.GeneralParam.RuntimeAccountParam.AdminUser, + DbBackupPwd: c.GeneralParam.RuntimeAccountParam.AdminPwd, + DbNames: c.dumpDbs, + DumpCmdFile: path.Join(cst.MysqldInstallPath, "bin", "mysqldump"), + Charset: c.charset, + MySQLDumpOption: dumpOption, + }, + OutputfileName: c.tmpDumpFile, + } + if err := dumper.Dump(); err != nil { + logger.Error("dump failed: %s", err.Error()) + return err + } + return nil +} + +func (c *ImportSchemaFromLocalSpiderComp) loadSchema() (err error) { + err = mysqlutil.ExecuteSqlAtLocal{ + IsForce: false, + Charset: c.charset, + NeedShowWarnings: false, + Host: c.Params.Host, + Port: c.Params.Port, + Socket: c.tdbctlSocket, + User: c.GeneralParam.RuntimeAccountParam.AdminUser, + Password: c.GeneralParam.RuntimeAccountParam.AdminPwd, + }.ExcuteSqlByMySQLClientOne(c.tmpDumpFile, "") + if err != nil { + logger.Error("执行导入schema文件:%s 失败:%s", c.tmpDumpFile, err.Error()) + return err + } + return nil +} diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index bcaeb16576..28de0afa7f 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -306,6 +306,7 @@ class DBActuatorTypeEnum(str, StructuredEnum): class DBActuatorActionEnum(str, StructuredEnum): Sysinit = EnumField("sysinit", _("sysinit")) Deploy = EnumField("deploy", _("deploy")) + AppendDeploy = EnumField("append-deploy", _("append-deploy")) GetBackupFile = EnumField("find-local-backup", _("find-local-backup")) RestoreSlave = EnumField("restore-dr", _("restore-dr")) RecoverBinlog = EnumField("recover-binlog", _("recover-binlog")) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py b/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py index e34d60863e..fcefe4f86f 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py @@ -493,6 +493,15 @@ def spider_slave_install_package(self, spider_version: str) -> list: f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}/{mysql_crond_pkg.path}", ] + def tdbctl_install_package(self) -> list: + tdbctl_pkg = Package.get_latest_package( + version=MediumEnum.Latest, pkg_type=MediumEnum.tdbCtl, db_type=DBType.MySQL + ) + return [ + f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}/{self.actuator_pkg.path}", + f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}/{tdbctl_pkg.path}", + ] + @staticmethod def get_spider_apps_package(): """ diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/migrate_spider_cluster_from_gcs.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/migrate_spider_cluster_from_gcs.py new file mode 100644 index 0000000000..d87a2c4183 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/migrate_spider_cluster_from_gcs.py @@ -0,0 +1,171 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import logging.config +from dataclasses import asdict +from typing import Dict, List, Optional + +from django.core.exceptions import ObjectDoesNotExist +from django.utils.crypto import get_random_string +from django.utils.translation import ugettext as _ + +from backend import env +from backend.configuration.constants import DBType +from backend.constants import IP_PORT_DIVIDER +from backend.db_meta.enums import ClusterType, TenDBClusterSpiderRole +from backend.db_meta.exceptions import ClusterNotExistException +from backend.db_meta.models import Cluster, StorageInstanceTuple +from backend.flow.consts import TDBCTL_USER +from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder +from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList +from backend.flow.engine.bamboo.scene.mysql.common.common_sub_flow import ( + build_repl_by_manual_input_sub_flow, + build_surrounding_apps_sub_flow, +) +from backend.flow.engine.bamboo.scene.spider.common.common_sub_flow import ( + build_apps_for_spider_sub_flow, + build_ctl_replication_with_gtid, +) +from backend.flow.plugins.components.collections.mysql.dns_manage import MySQLDnsManageComponent +from backend.flow.plugins.components.collections.mysql.exec_actuator_script import ExecuteDBActuatorScriptComponent +from backend.flow.plugins.components.collections.mysql.mysql_os_init import MySQLOsInitComponent, SysInitComponent +from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent +from backend.flow.plugins.components.collections.spider.add_system_user_in_cluster import ( + AddSystemUserInClusterComponent, +) +from backend.flow.plugins.components.collections.spider.spider_db_meta import SpiderDBMetaComponent +from backend.flow.utils.mysql.mysql_act_dataclass import ( + AddSpiderSystemUserKwargs, + CreateDnsKwargs, + DBMetaOPKwargs, + DownloadMediaKwargs, + ExecActuatorKwargs, +) +from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload +from backend.flow.utils.mysql.mysql_context_dataclass import SpiderApplyManualContext +from backend.flow.utils.spider.spider_act_dataclass import InstanceTuple, ShardInfo +from backend.flow.utils.spider.spider_db_meta import SpiderDBMeta + + +class MigrateClusterFromGcsFlow(object): + """ + migrate cluster from gcs + 1、追加部署中控 + 2、授权 + 3、导入表结构 + 4、检查表结构一致性 + """ + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + @param root_id : 任务流程定义的root_id + @param data : 单据传递参数 + { + "bk_cloud_id": , + "infos":[ + { + "cluster_id": cluster_id + } + ] + } + """ + self.root_id = root_id + self.data = data + self.cluster_type = ClusterType.TenDBCluster + self.bk_cloud_id = 0 + if self.data.get("bk_cloud_id"): + self.bk_cloud_id = self.data["bk_cloud_id"] + + # 一个单据自动生成同一份随机密码, 中控实例需要，不需要内部来维护,每次部署随机生成一次 + self.tdbctl_pass = get_random_string(length=10) + + def run(self): + # 初始化流程 + pipeline = Builder(root_id=self.root_id, data=self.data) + migrate_pipeline = SubBuilder(root_id=self.root_id, data=self.data) + # 拼接执行原子任务活动节点需要的通用的私有参数结构体, 减少代码重复率，但引用时注意内部参数值传递的问题 + exec_act_kwargs = ExecActuatorKwargs( + bk_cloud_id=int(self.data["bk_cloud_id"]), + cluster_type=ClusterType.TenDBCluster, + ) + + for job in self.data["infos"]: + try: + cluster_obj = Cluster.objects.get( + pk=job["cluster_id"], bk_biz_id=self.data["bk_biz_id"], cluster_type=self.cluster_type + ) + except ObjectDoesNotExist: + raise ClusterNotExistException(cluster_type=self.cluster_type, cluster_id=job["cluster_id"]) + + ctl_objs = cluster_obj.proxyinstance_set.all() + master_spider_ips = [c.machine.ip for c in ctl_objs] + logging.info("master_spider_ips: %s" % [c.machine.ip for c in ctl_objs]) + if len(master_spider_ips) < 2: + raise Exception(_("至少需要2个以上的spider节点")) + ctl_master = master_spider_ips[0] + ctl_slaves = master_spider_ips[1:] + ctl_port = ctl_objs[0].port + 1000 + # 给spider节点下发tdbctl 介质 0 + migrate_pipeline.add_act( + act_name=_("下发tdbCtl介质包"), + act_component_code=TransFileComponent.code, + kwargs=asdict( + DownloadMediaKwargs( + bk_cloud_id=self.bk_cloud_id, + exec_ip=master_spider_ips, + file_list=GetFileList(db_type=DBType.MySQL).tdbctl_install_package(), + ) + ), + ) + + acts_list = [] + # 这里中控实例安装和spider机器复用的 + for ctl_ip in master_spider_ips: + exec_act_kwargs.exec_ip = ctl_ip + exec_act_kwargs.cluster = { + "immutable_domain": cluster_obj.immute_domain, + "ctl_port": ctl_port, + "ctl_charset": job["ctl_charset"], + } + exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.get_append_deploy_spider_ctl_payload.__name__ + acts_list.append( + { + "act_name": _("安装Tdbctl集群中控实例"), + "act_component_code": ExecuteDBActuatorScriptComponent.code, + "kwargs": asdict(exec_act_kwargs), + } + ) + + migrate_pipeline.add_parallel_acts(acts_list=acts_list) + # 构建spider中控集群 + migrate_pipeline.add_sub_pipeline( + sub_flow=build_ctl_replication_with_gtid( + root_id=self.root_id, + parent_global_data=self.data, + bk_cloud_id=int(self.data["bk_cloud_id"]), + ctl_primary=f"{ctl_master}{IP_PORT_DIVIDER}{ctl_port}", + ctl_secondary_list=[{"ip": value} for value in ctl_slaves], + ) + ) + # 内部集群节点之间授权 + migrate_pipeline.add_act( + act_name=_("集群内部节点间授权"), + act_component_code=AddSystemUserInClusterComponent.code, + kwargs=asdict( + AddSpiderSystemUserKwargs(ctl_master_ip=ctl_master, user=TDBCTL_USER, passwd=self.tdbctl_pass) + ), + ) + + # 运行流程 + pipeline.add_sub_pipeline( + sub_flow=migrate_pipeline.build_sub_process(sub_name=_("{}集群中控追加部署").format(cluster_obj.name)) + ) + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/controller/spider.py b/dbm-ui/backend/flow/engine/controller/spider.py index 99a7ef6c11..f4c94fb29c 100644 --- a/dbm-ui/backend/flow/engine/controller/spider.py +++ b/dbm-ui/backend/flow/engine/controller/spider.py @@ -10,6 +10,7 @@ from backend.db_meta.enums import ClusterType from backend.flow.engine.bamboo.scene.spider.import_sqlfile_flow import ImportSQLFlow +from backend.flow.engine.bamboo.scene.spider.migrate_spider_cluster_from_gcs import MigrateClusterFromGcsFlow from backend.flow.engine.bamboo.scene.spider.remote_local_slave_recover import TenDBRemoteSlaveLocalRecoverFlow from backend.flow.engine.bamboo.scene.spider.remote_master_fail_over import RemoteMasterFailOverFlow from backend.flow.engine.bamboo.scene.spider.remote_master_slave_swtich import RemoteMasterSlaveSwitchFlow @@ -208,3 +209,10 @@ def reduce_spider_mnt_scene(self): """ flow = TenDBClusterReduceMNTFlow(root_id=self.root_id, data=self.ticket_data) flow.reduce_spider_mnt() + + def migrate_spider_cluster_from_gcs(self): + """ + 转移spider cluster 从Gcs到dbm 系统需要的操作 + """ + flow = MigrateClusterFromGcsFlow(root_id=self.root_id, data=self.ticket_data) + flow.run() diff --git a/dbm-ui/backend/flow/urls.py b/dbm-ui/backend/flow/urls.py index fa458b9c3e..ab27e78aa0 100644 --- a/dbm-ui/backend/flow/urls.py +++ b/dbm-ui/backend/flow/urls.py @@ -51,6 +51,7 @@ from backend.flow.views.kafka_replace import ReplaceKafkaSceneApiView from backend.flow.views.kafka_scale_up import ScaleUpKafkaSceneApiView from backend.flow.views.kafka_shrink import ShrinkKafkaSceneApiView +from backend.flow.views.migrate_spider_cluster_from_gcs import MigrateSpiderClusterFromGcsView from backend.flow.views.migrate_views.es_fake_apply import FakeInstallEsSceneApiView from backend.flow.views.migrate_views.hdfs_fake_apply import FakeInstallHdfsSceneApiView from backend.flow.views.migrate_views.influxdb_fake_apply import FakeInstallInfluxdbSceneApiView @@ -374,4 +375,6 @@ url("^scene/switch_tbinlogumper$", SwitchTBinlogDumperSceneApiView.as_view()), url("^scene/tendbha_standardize$", TenDBHAStandardizeView.as_view()), url("^scene/mysql_open_area$", MysqlOpenAreaSceneApiView.as_view()), + # migrate + url("^scene/migrate_spider_cluster_from_gcs$", MigrateSpiderClusterFromGcsView.as_view()), ] diff --git a/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py b/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py index e47c42864e..a9eb3ea2dc 100644 --- a/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py +++ b/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py @@ -262,6 +262,40 @@ def get_install_slave_spider_payload(self, **kwargs): content["payload"]["extend"]["mycnf_configs"][key]["mysqld"].update(slave_config) return content + def get_append_deploy_spider_ctl_payload(self, **kwargs): + """ + 拼接spider-ctl节点添加单实例的payload + """ + ctl_charset = self.cluster["ctl_charset"] + + ctl_pkg = Package.get_latest_package(version=MediumEnum.Latest, pkg_type=MediumEnum.tdbCtl) + version_no = get_mysql_real_version(ctl_pkg.name) + + drs_account, dbha_account = self.get_super_account() + return { + "db_type": DBActuatorTypeEnum.SpiderCtl.value, + "action": DBActuatorActionEnum.AppendDeploy.value, + "payload": { + "general": {"runtime_account": self.account}, + "extend": { + "host": kwargs["ip"], + "pkg": ctl_pkg.name, + "pkg_md5": ctl_pkg.md5, + "mysql_version": version_no, + "charset": ctl_charset, + "inst_mem": 0, + "ports": [self.cluster["ctl_port"]], + "super_account": drs_account, + "dbha_account": dbha_account, + "mycnf_configs": { + self.cluster["ctl_port"]: self.__get_mysql_config( + immutable_domain=self.cluster["immutable_domain"], db_version="Tdbctl" + ) + }, + }, + }, + } + def get_install_spider_ctl_payload(self, **kwargs): """ 拼接spider-ctl节点安装的payload, ctl是单机单实例, 所以代码兼容多实例传入 diff --git a/dbm-ui/backend/flow/views/migrate_spider_cluster_from_gcs.py b/dbm-ui/backend/flow/views/migrate_spider_cluster_from_gcs.py new file mode 100644 index 0000000000..12d11542b4 --- /dev/null +++ b/dbm-ui/backend/flow/views/migrate_spider_cluster_from_gcs.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +import uuid + +from django.utils.translation import ugettext as _ +from rest_framework.response import Response + +from backend.flow.engine.controller.spider import SpiderController +from backend.flow.views.base import FlowTestView + +logger = logging.getLogger("root") + + +class MigrateSpiderClusterFromGcsView(FlowTestView): + """ + api: /apis/v1/flow/scene/migrate_spider_cluster_from_gcs + """ + + @staticmethod + def post(request): + logger.info(_("开始执行迁移后置操作,部署tdbctl中控")) + + root_id = uuid.uuid1().hex + logger.info("define root_id: {}".format(root_id)) + + c = SpiderController(root_id=root_id, ticket_data=request.data) + c.migrate_spider_cluster_from_gcs() + return Response({"root_id": root_id})