diff --git a/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_replace.go b/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_replace.go index 1be1ad55a4..228fac305a 100644 --- a/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_replace.go +++ b/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_replace.go @@ -137,19 +137,21 @@ func (r *MongoDReplace) Init(runtime *jobruntime.JobGenericRuntime) error { r.StatusCh = make(chan int, 1) // 获取源端的配置信息 - _, _, _, hidden, priority, _, err := common.GetNodeInfo(r.Mongo, r.PrimaryIP, r.PrimaryPort, - r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, r.ConfParams.SourceIP, r.ConfParams.SourcePort) - if err != nil { - return err + if r.ConfParams.SourceIP != "" { + _, _, _, hidden, priority, _, err := common.GetNodeInfo(r.Mongo, r.PrimaryIP, r.PrimaryPort, + r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, r.ConfParams.SourceIP, r.ConfParams.SourcePort) + if err != nil { + return err + } + r.TargetPriority = priority + r.TargetHidden = hidden } - r.TargetHidden = hidden if r.ConfParams.TargetHidden == "0" { r.TargetHidden = false } else if r.ConfParams.TargetHidden == "1" { r.TargetHidden = true } - r.TargetPriority = priority if r.ConfParams.TargetPriority != "" { r.TargetPriority, _ = strconv.Atoi(r.ConfParams.TargetPriority) } @@ -359,7 +361,7 @@ func (r *MongoDReplace) checkTargetStatusAndRemoveSource() error { case <-time.After(50 * time.Second): return fmt.Errorf("check target status timeout") case status := <-r.StatusCh: - if status == 2 && r.ConfParams.SourceDown == false { + if status == 2 && r.ConfParams.SourceDown == false && r.ConfParams.SourceIP != "" { if err := r.shutdownSourceProcess(); err != nil { return err } diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/__init__.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_scale_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_scale_node.py new file mode 100644 index 0000000000..5cdb9e2abe --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_scale_node.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task import ( + cluster_increase_node, + cluster_reduce_node, + replicaset_reduce_node, + replicaset_set_increase_node, +) +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoScaleNodeFlow(object): + """MongoDB变更shard节点数flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表，是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_file_path() + + def multi_cluster_scale_node_flow(self, increase: bool): + """ + multi cluster scale node流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + sub_pipelines = [] + # 复制集增减节点——子流程并行 + if self.data["infos"][ClusterType.MongoReplicaSet.value]: + for replicaset_set in self.data["infos"][ClusterType.MongoReplicaSet.value]: + if increase: + sub_pipline = replicaset_set_increase_node( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + info=replicaset_set, + ) + sub_pipelines.append(sub_pipline) + else: + sub_pipline = replicaset_reduce_node( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + info=replicaset_set, + cluster=False, + ) + sub_pipelines.append(sub_pipline) + # cluster的shard增减节点——子流程并行 + if self.data["infos"][ClusterType.MongoShardedCluster.value]: + for cluster in self.data["infos"][ClusterType.MongoShardedCluster.value]: + if increase: + sub_pipline = cluster_increase_node( + root_id=self.root_id, ticket_data=self.data, sub_kwargs=self.get_kwargs, info=cluster + ) + else: + sub_pipline = cluster_reduce_node( + root_id=self.root_id, ticket_data=self.data, sub_kwargs=self.get_kwargs, info=cluster + ) + sub_pipelines.append(sub_pipline) + + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py index 14f5e260d2..669848463e 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py @@ -10,17 +10,24 @@ """ +from .cluster_increase_node import cluster_increase_node +from .cluster_reduce_node import cluster_reduce_node from .cluster_replace import cluster_replace from .cluster_scale import cluster_scale from .deinstall import deinstall from .exec_script import exec_script +from .increase_mongod import increase_mongod from .increase_mongos import increase_mongos from .instance_restart import instance_restart from .mongod_replace import mongod_replace from .mongos_install import mongos_install from .mongos_replace import mongos_replace +from .reduce_mongod import reduce_mongod from .reduce_mongos import reduce_mongos from .replicaset_install import replicaset_install +from .replicaset_reduce_node import replicaset_reduce_node from .replicaset_replace import replicaset_replace from .replicaset_scale import replicaset_scale +from .replicaset_set_increase_node import replicaset_set_increase_node +from .replicaset_set_increase_node_by_ip import replicaset_set_increase_node_by_ip from .user import user diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py new file mode 100644 index 0000000000..648fd43537 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +from .replicaset_set_increase_node_by_ip import replicaset_set_increase_node_by_ip + + +def cluster_increase_node(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict) -> SubBuilder: + """ + cluster增加节点流程 + info 表示同机器多cluster信息 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 设置变量 + sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] + sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoShardedCluster.value + info["target"] = info["add_shard_nodes"][0] + + # 计算cacheSize oplogSize + sub_get_kwargs.calc_param_replace(info=info, instance_num=info["node_replica_count"]) + + # 获取主机信息 + sub_get_kwargs.get_host_increase_node(info=info) + + # 介质下发 + kwargs = sub_get_kwargs.get_send_media_kwargs(media_type="all") + sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建原子任务执行目录 + kwargs = sub_get_kwargs.get_create_dir_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-创建原子任务执行目录"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 机器初始化 + kwargs = sub_get_kwargs.get_os_init_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-机器初始化"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 获取集群信息并以IP为维度计算对应关系 + sub_get_kwargs.calc_increase_node(info=info) + + # 获取mongos信息 + mongos_host = sub_get_kwargs.payload["mongos_nodes"][0] + sub_get_kwargs.payload["nodes"] = [ + {"ip": mongos_host["ip"], "port": mongos_host["port"], "bk_cloud_id": mongos_host["bk_cloud_id"]} + ] + + # 获取密码 + get_password = {} + get_password["usernames"] = sub_get_kwargs.manager_users + sub_get_kwargs.payload["passwords"] = sub_get_kwargs.get_password_from_db(info=get_password)["passwords"] + + # 获取key_file + sub_get_kwargs.cluster_type = sub_get_kwargs.payload["cluster_type"] + sub_get_kwargs.payload["key_file"] = sub_get_kwargs.get_key_file( + cluster_name=sub_get_kwargs.payload["cluster_name"] + ) + + # 以IP为维度增加node——子流程并行 + sub_pipelines = [] + for add_shard_node, shards_instance_relationship in sub_get_kwargs.payload[ + "shards_instance_relationships_by_ip" + ].items(): + sub_get_kwargs.payload["add_shard_node"] = add_shard_node + sub_sub_pipeline = replicaset_set_increase_node_by_ip( + root_id=root_id, + ticket_data=ticket_data, + sub_kwargs=sub_get_kwargs, + info=shards_instance_relationship, + cluster=True, + ) + sub_pipelines.append(sub_sub_pipeline) + sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}增加节点".format(sub_get_kwargs.payload["cluster_name"]))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_reduce_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_reduce_node.py new file mode 100644 index 0000000000..8b37ee2d1f --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_reduce_node.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +from .replicaset_reduce_node import replicaset_reduce_node + + +def cluster_reduce_node(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict) -> SubBuilder: + """ + cluster减少节点流程 + info 表示同机器多cluster信息 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 设置变量 + sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] + sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoShardedCluster.value + + # 获取集群信息并计算对应关系 + sub_get_kwargs.calc_reduce_node(info=info) + + # 介质下发 + kwargs = sub_get_kwargs.get_send_media_kwargs(media_type="actuator") + sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建原子任务执行目录 + kwargs = sub_get_kwargs.get_create_dir_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-创建原子任务执行目录"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 获取mongos信息 + mongos_host = sub_get_kwargs.payload["mongos_nodes"][0] + sub_get_kwargs.payload["nodes"] = [ + {"ip": mongos_host["ip"], "port": mongos_host["port"], "bk_cloud_id": mongos_host["bk_cloud_id"]} + ] + + # 获取密码 + get_password = {} + get_password["usernames"] = sub_get_kwargs.manager_users + sub_get_kwargs.payload["passwords"] = sub_get_kwargs.get_password_from_db(info=get_password)["passwords"] + + # shard进行减少node——子流程并行 + sub_pipelines = [] + for db_instances in sub_get_kwargs.payload["shards_instance_relationships"].values(): + sub_get_kwargs.payload["db_instances"] = db_instances + sub_sub_pipeline = replicaset_reduce_node( + root_id=root_id, + ticket_data=ticket_data, + sub_kwargs=sub_get_kwargs, + info={}, + cluster=True, + ) + sub_pipelines.append(sub_sub_pipeline) + sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}增加节点".format(sub_get_kwargs.payload["cluster_name"]))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/increase_mongod.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/increase_mongod.py new file mode 100644 index 0000000000..51917f0c77 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/increase_mongod.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MongoDBManagerUser +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.add_domain_to_dns import ExecAddDomainToDnsOperationComponent +from backend.flow.plugins.components.collections.mongodb.add_password_to_db import ( + ExecAddPasswordToDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def increase_mongod( + root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict, cluster_role: str +) -> SubBuilder: + """ + replicaset 增加节点流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 设置参数 + node = sub_get_kwargs.db_instance + sub_get_kwargs.replicaset_info["nodes"] = [ + { + "ip": node["ip"], + "domain": node.get("domain", ""), + "bk_cloud_id": node["bk_cloud_id"], + "port": node["port"], + } + ] + if not cluster_role: + # 获取集群信息并计算对应关系 + sub_get_kwargs.get_cluster_info_deinstall(cluster_id=info["cluster_id"]) + + # 获取密码 + get_password = {} + get_password["usernames"] = sub_get_kwargs.manager_users + sub_get_kwargs.payload["passwords"] = sub_get_kwargs.get_password_from_db(info=get_password)["passwords"] + + # 获取key_file + sub_get_kwargs.cluster_type = sub_get_kwargs.payload["cluster_type"] + sub_get_kwargs.replicaset_info["key_file"] = sub_get_kwargs.get_key_file( + cluster_name=sub_get_kwargs.payload["cluster_name"] + ) + + # 设置参数 + sub_get_kwargs.replicaset_info["set_id"] = sub_get_kwargs.payload["set_id"] + shard_name = sub_get_kwargs.payload["cluster_name"] + operation_node = sub_get_kwargs.payload["nodes"][0] + else: + shard_name = node["seg_range"] + sub_get_kwargs.replicaset_info["set_id"] = shard_name + for shard in sub_get_kwargs.payload["shards_nodes"]: + if shard["shard"] == shard_name: + sub_get_kwargs.payload["operation_node"] = shard["nodes"][0] + break + operation_node = sub_get_kwargs.payload["operation_node"] + + sub_get_kwargs.replicaset_info["port"] = node["port"] + # 新节点作为执行操作节点 + add_node_info = { + "exec_ip": node["ip"], + "exec_bk_cloud_id": node["bk_cloud_id"], + "ip": operation_node["ip"], + "port": operation_node["port"], + "bk_cloud_id": operation_node["bk_cloud_id"], + "admin_user": MongoDBManagerUser.DbaUser.value, + "admin_password": sub_get_kwargs.payload["passwords"][MongoDBManagerUser.DbaUser.value], + "target": node, + } + + # mongod安装 + kwargs = sub_get_kwargs.get_install_mongod_kwargs(node=node, cluster_role=cluster_role) + sub_pipeline.add_act( + act_name=_("MongoDB-{}-mongod安装".format(node["ip"])), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + # 添加到复制集中 + kwargs = sub_get_kwargs.get_increase_node_kwargs(info=add_node_info) + sub_pipeline.add_act( + act_name=_("MongoDB-添加node"), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + # 添加新增实例dns + if not cluster_role: + kwargs = sub_get_kwargs.get_add_domain_to_dns_kwargs(cluster=False) + sub_pipeline.add_act( + act_name=_("MongoDB-添加新实例的domain到dns"), + act_component_code=ExecAddDomainToDnsOperationComponent.code, + kwargs=kwargs, + ) + + # 保存新实例密码 + kwargs = sub_get_kwargs.get_add_password_to_db_kwargs( + usernames=sub_get_kwargs.manager_users, + info=sub_get_kwargs.replicaset_info, + ) + # 用户获取密码 + kwargs["passwords"] = sub_get_kwargs.payload["passwords"] + # 是否是部署单据 + kwargs["create"] = False + sub_pipeline.add_act( + act_name=_("MongoDB-保存新实例的dba用户及额外管理用户密码"), + act_component_code=ExecAddPasswordToDBOperationComponent.code, + kwargs=kwargs, + ) + + return sub_pipeline.build_sub_process( + sub_name=_("MongoDB--{}增加node{}:{}".format(shard_name, node["ip"], node["port"])) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/reduce_mongod.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/reduce_mongod.py new file mode 100644 index 0000000000..182a30a453 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/reduce_mongod.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MongoDBInstanceType, MongoDBManagerUser +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.delete_domain_from_dns import ( + ExecDeleteDomainFromDnsOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.delete_password_from_db import ( + ExecDeletePasswordFromDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.mongodb_scale_repls_meta import MongoScaleReplsMetaComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def reduce_mongod(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, cluster: bool) -> SubBuilder: + """ + replicaset 减少节点流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 设置参数 + node = sub_get_kwargs.db_instance + sub_get_kwargs.replicaset_info = {} + sub_get_kwargs.replicaset_info["nodes"] = [ + { + "ip": node["ip"], + "domain": node.get("domain", ""), + "bk_cloud_id": node["bk_cloud_id"], + "port": node["port"], + } + ] + sub_get_kwargs.payload["nodes"] = [] + sub_get_kwargs.payload["nodes"].append(node) + if cluster: + sub_get_kwargs.payload["mongos_nodes"] = [] + sub_get_kwargs.payload["config_nodes"] = [] + sub_get_kwargs.payload["shards_nodes"] = [] + sub_get_kwargs.payload["shards_nodes"].append({"nodes": sub_get_kwargs.payload["nodes"]}) + else: + # 获取密码 + get_password = {} + get_password["usernames"] = sub_get_kwargs.manager_users + sub_get_kwargs.payload["passwords"] = sub_get_kwargs.get_password_from_db(info=get_password)["passwords"] + + # 移除节点作为执行操作节点 + remove_node_info = { + "exec_ip": node["ip"], + "exec_bk_cloud_id": node["bk_cloud_id"], + "ip": node["ip"], + "port": node["port"], + "bk_cloud_id": node["bk_cloud_id"], + "admin_user": MongoDBManagerUser.DbaUser.value, + "admin_password": sub_get_kwargs.payload["passwords"][MongoDBManagerUser.DbaUser.value], + "source": node, + } + + # 从复制集中移除 + kwargs = sub_get_kwargs.get_reduce_node_kwargs(info=remove_node_info) + sub_pipeline.add_act( + act_name=_("MongoDB-移除node"), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + # mongod下架 + kwargs = sub_get_kwargs.get_mongo_deinstall_kwargs( + node_info=node, + instance_type=MongoDBInstanceType.MongoD.value, + nodes_info=[node], + force=True, + ) + sub_pipeline.add_act( + act_name=_("MongoDB-{}-mongod下架".format(node["ip"])), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + # 实例dns删除 + if not cluster: + kwargs = sub_get_kwargs.get_delete_domain_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-删除node的domain指向"), + act_component_code=ExecDeleteDomainFromDnsOperationComponent.code, + kwargs=kwargs, + ) + + # 删除实例密码 + kwargs = sub_get_kwargs.get_delete_pwd_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-删除node的dba用户及额外管理用户密码"), + act_component_code=ExecDeletePasswordFromDBOperationComponent.code, + kwargs=kwargs, + ) + + # 修改meta + kwargs = sub_get_kwargs.get_meta_scale_node_kwargs(info=[node], increase=False) + sub_pipeline.add_act( + act_name=_("MongoDB-修改meta"), + act_component_code=MongoScaleReplsMetaComponent.code, + kwargs=kwargs, + ) + + return sub_pipeline.build_sub_process( + sub_name=_("MongoDB--{}减少node{}:{}".format(node["cluster_name"], node["ip"], node["port"])) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_reduce_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_reduce_node.py new file mode 100644 index 0000000000..05f5194ed6 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_reduce_node.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +from .reduce_mongod import reduce_mongod + + +def replicaset_reduce_node( + root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict, cluster: bool +) -> SubBuilder: + """ + replicaset减少节点流程 + info 表示replicaset信息 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + if not cluster: + # 设置变量 + sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] + sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoReplicaSet.value + + # 获取集群信息并计算对应关系 + sub_get_kwargs.calc_reduce_node(info=info) + + # 介质下发 + kwargs = sub_get_kwargs.get_send_media_kwargs(media_type="actuator") + sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建原子任务执行目录 + kwargs = sub_get_kwargs.get_create_dir_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-创建原子任务执行目录"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 复制集进行减少node——子流程串行 + sub_pipelines = [] + for db_instance in sub_get_kwargs.payload["db_instances"]: + sub_get_kwargs.db_instance = db_instance + sub_sub_pipeline = reduce_mongod( + root_id=root_id, + ticket_data=ticket_data, + sub_kwargs=sub_get_kwargs, + cluster=cluster, + ) + sub_pipelines.append(sub_sub_pipeline) + sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}减少节点".format("replicaset"))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py new file mode 100644 index 0000000000..2f7fca4756 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +from .replicaset_set_increase_node_by_ip import replicaset_set_increase_node_by_ip + + +def replicaset_set_increase_node( + root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict +) -> SubBuilder: + """ + 同机器多replicaset增加节点流程 + info 表示同机器多replicaset信息 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 设置变量 + sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] + info["target"] = info["add_shard_nodes"][0] + + # 计算cacheSize oplogSize + sub_get_kwargs.calc_param_replace(info=info, instance_num=len(info["cluster_ids"])) + + # 获取主机信息 + sub_get_kwargs.get_host_increase_node(info=info) + + # 介质下发 + kwargs = sub_get_kwargs.get_send_media_kwargs(media_type="all") + sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建原子任务执行目录 + kwargs = sub_get_kwargs.get_create_dir_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-创建原子任务执行目录"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 机器初始化 + kwargs = sub_get_kwargs.get_os_init_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-机器初始化"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 以IP为维度增加node——子流程并行 + sub_sub_pipelines = [] + for node_index, add_shard_node in enumerate(info["add_shard_nodes"]): + # 机器的索引即是增加node的索引 + info["add_shard_node"] = add_shard_node + info["add_shard_node"]["node_index"] = node_index + sub_sub_pipeline = replicaset_set_increase_node_by_ip( + root_id=root_id, + ticket_data=ticket_data, + sub_kwargs=sub_get_kwargs, + info=info, + cluster=False, + ) + sub_sub_pipelines.append(sub_sub_pipeline) + sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_sub_pipelines) + + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}增加node".format("replicaset"))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node_by_ip.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node_by_ip.py new file mode 100644 index 0000000000..4be5f59e7f --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node_by_ip.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.consts import MongoDBClusterRole +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.mongodb_scale_repls_meta import MongoScaleReplsMetaComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +from .increase_mongod import increase_mongod + + +def replicaset_set_increase_node_by_ip( + root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict, cluster: bool +) -> SubBuilder: + """ + 同机器多replicaset增加节点流程 + info 表示同机器多replicaset信息 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取集群信息并计算对应关系 + if not cluster: + sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoReplicaSet.value + sub_get_kwargs.calc_increase_node(info=info) + ip = info["add_shard_node"]["ip"] + cluster_role = "" + else: + sub_get_kwargs.payload["replicaset_set"] = info + ip = sub_get_kwargs.payload["add_shard_node"] + cluster_role = MongoDBClusterRole.ShardSvr.value + + # 复制集维度增加node——子流程并行 + sub_sub_pipelines = [] + for replicaset in sub_get_kwargs.payload["replicaset_set"]: + sub_get_kwargs.db_instance = replicaset + sub_sub_pipeline = increase_mongod( + root_id=root_id, + ticket_data=ticket_data, + sub_kwargs=sub_get_kwargs, + info=replicaset, + cluster_role=cluster_role, + ) + sub_sub_pipelines.append(sub_sub_pipeline) + sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_sub_pipelines) + + # 修改meta——串行 + if not cluster: + for scale_out in sub_get_kwargs.payload["scale_outs"]: + kwargs = sub_get_kwargs.get_meta_scale_node_kwargs(info=[scale_out], increase=True) + sub_pipeline.add_act( + act_name=_("MongoDB-修改meta-{}:{}".format(scale_out["ip"], str(scale_out["port"]))), + act_component_code=MongoScaleReplsMetaComponent.code, + kwargs=kwargs, + ) + + else: + kwargs = sub_get_kwargs.get_meta_scale_node_kwargs( + info=sub_get_kwargs.payload["scale_out_instances_by_ip"][sub_get_kwargs.payload["add_shard_node"]], + increase=True, + ) + sub_pipeline.add_act( + act_name=_("MongoDB-修改meta"), + act_component_code=MongoScaleReplsMetaComponent.code, + kwargs=kwargs, + ) + + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}增加node".format(ip))) diff --git a/dbm-ui/backend/flow/engine/controller/mongodb.py b/dbm-ui/backend/flow/engine/controller/mongodb.py index 1d1356fcb8..8ef59aab9a 100644 --- a/dbm-ui/backend/flow/engine/controller/mongodb.py +++ b/dbm-ui/backend/flow/engine/controller/mongodb.py @@ -19,6 +19,7 @@ from backend.flow.engine.bamboo.scene.mongodb.mongodb_remove_ns import MongoRemoveNsFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_replace import MongoReplaceFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_restore import MongoRestoreFlow +from backend.flow.engine.bamboo.scene.mongodb.mongodb_scale_node import MongoScaleNodeFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_scale_storage import MongoScaleFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_user import MongoUserFlow from backend.flow.engine.controller.base import BaseController @@ -147,3 +148,19 @@ def scale_cluster(self): flow = MongoScaleFlow(root_id=self.root_id, data=self.ticket_data) flow.multi_cluster_scale_flow() + + def increase_node(self): + """ + 增加node数 + """ + + flow = MongoScaleNodeFlow(root_id=self.root_id, data=self.ticket_data) + flow.multi_cluster_scale_node_flow(increase=True) + + def reduce_node(self): + """ + 减少node数 + """ + + flow = MongoScaleNodeFlow(root_id=self.root_id, data=self.ticket_data) + flow.multi_cluster_scale_node_flow(increase=False) diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/mongodb_scale_repls_meta.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/mongodb_scale_repls_meta.py index b63f9a50a3..51cbd31731 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mongodb/mongodb_scale_repls_meta.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/mongodb_scale_repls_meta.py @@ -183,7 +183,7 @@ def scale_out_repls(self, cluster: Cluster, scale_list: Dict, created_by: str): machine_exist = Machine.objects.filter( bk_biz_id=cluster.bk_biz_id, ip=scale_item["ip"], bk_cloud_id=cluster.bk_cloud_id ).exists() - if scale_item.get("reuse_machine", False) and not machine_exist: + if not machine_exist: api.machine.create( machines=[ { @@ -197,16 +197,6 @@ def scale_out_repls(self, cluster: Cluster, scale_list: Dict, created_by: str): bk_cloud_id=cluster.bk_cloud_id, creator=created_by, ) - else: - raise Exception( - "machine will not create reuseFlag:{} and machineExist:{} bizID:{};IP:{};CloudID:{}".format( - scale_item.get("reuse_machine", False), - machine_exist, - cluster.bk_biz_id, - scale_item["ip"], - cluster.bk_cloud_id, - ) - ) mongo_obj = api.storage_instance.create( instances=[{"ip": scale_item["ip"], "port": scale_item["port"], "instance_role": role}], status=InstanceStatus.RUNNING.value, @@ -231,8 +221,9 @@ def scale_out_repls(self, cluster: Cluster, scale_list: Dict, created_by: str): cluster.storageinstance_set.add(mongo_obj) # storageinstance_bind_entry 增加记录 - tmp_entries = m1_obj.bind_entry.all() - mongo_obj.bind_entry.add(*tmp_entries) + if m1_obj.cluster_type != ClusterType.MongoReplicaSet.value: + tmp_entries = m1_obj.bind_entry.all() + mongo_obj.bind_entry.add(*tmp_entries) is_increment = False if m1_obj.cluster_type == ClusterType.MongoReplicaSet.value: @@ -247,6 +238,9 @@ def scale_out_repls(self, cluster: Cluster, scale_list: Dict, created_by: str): cluster_entry.storageinstance_set.add( StorageInstance.objects.get(machine__ip=scale_item["ip"], port=scale_item["port"]) ) + logger.info( + "add domain {}:{} 2 add_node:info::: {} done".format(mongo_obj, cluster.immute_domain, scale_item) + ) cluster_entry.save() # 转移模块 MongoDBCCTopoOperator(cluster).transfer_instances_to_cluster_module( diff --git a/dbm-ui/backend/flow/urls.py b/dbm-ui/backend/flow/urls.py index 4b8f1a4ddb..4ef9d6743a 100644 --- a/dbm-ui/backend/flow/urls.py +++ b/dbm-ui/backend/flow/urls.py @@ -73,14 +73,16 @@ ClusterInstallApiView, MongoBackupApiView, MongoDBCreateUserView, - MongoDBDeInstallSView, + MongoDBDeInstallView, MongoDBDeleteUserView, MongoDBExecScriptView, MongoDBIncreaseMongoSView, + MongoDBIncreaseNodeView, MongoDBInstanceRestartView, MongoDBReduceMongoSView, + MongoDBReduceNodeView, MongoDBReplaceView, - MongoDBScaleSView, + MongoDBScaleView, MongoFakeInstallApiView, MultiReplicasetInstallApiView, ) @@ -300,8 +302,10 @@ url(r"^scene/multi_hosts_replace$", MongoDBReplaceView.as_view()), url(r"^scene/multi_cluster_increase_mongos$", MongoDBIncreaseMongoSView.as_view()), url(r"^scene/multi_cluster_reduce_mongos$", MongoDBReduceMongoSView.as_view()), - url(r"^scene/cluster_deinstall$", MongoDBDeInstallSView.as_view()), - url(r"^scene/cluster_scale$", MongoDBScaleSView.as_view()), + url(r"^scene/cluster_deinstall$", MongoDBDeInstallView.as_view()), + url(r"^scene/cluster_scale$", MongoDBScaleView.as_view()), + url(r"^scene/multi_cluster_increase_node$", MongoDBIncreaseNodeView.as_view()), + url(r"^scene/multi_cluster_reduce_node$", MongoDBReduceNodeView.as_view()), # mongodb end # mysql upgrade url(r"^scene/upgrade_mysql_proxy$", UpgradeMySQLProxySceneApiView.as_view()), diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py index e0705fe493..eacd6575a8 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py @@ -80,6 +80,20 @@ def __init__(self): MongoDBManagerUser.MonitorUser.value, MongoDBManagerUser.AppMonitorUser.value, ] + # 实例角色 + self.instance_role = [ + InstanceRole.MONGO_M1.value, + InstanceRole.MONGO_M2.value, + InstanceRole.MONGO_M3.value, + InstanceRole.MONGO_M4.value, + InstanceRole.MONGO_M5.value, + InstanceRole.MONGO_M6.value, + InstanceRole.MONGO_M7.value, + InstanceRole.MONGO_M8.value, + InstanceRole.MONGO_M9.value, + InstanceRole.MONGO_M10.value, + InstanceRole.MONGO_BACKUP.value, + ] def __get_define_config(self, namespace: str, conf_file: str, conf_type: str) -> Any: """获取一些全局的参数配置""" @@ -436,19 +450,7 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: "region": self.payload["city"], "db_module_id": 0, } - instance_role = [ - InstanceRole.MONGO_M1, - InstanceRole.MONGO_M2, - InstanceRole.MONGO_M3, - InstanceRole.MONGO_M4, - InstanceRole.MONGO_M5, - InstanceRole.MONGO_M6, - InstanceRole.MONGO_M7, - InstanceRole.MONGO_M8, - InstanceRole.MONGO_M9, - InstanceRole.MONGO_M10, - InstanceRole.MONGO_BACKUP, - ] + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: info["cluster_type"] = ClusterType.MongoReplicaSet.value info["skip_machine"] = replicaset_info["skip_machine"] @@ -466,7 +468,7 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: if index == len(replicaset_info["nodes"]) - 1: info["storages"].append( { - "role": instance_role[-1], + "role": self.instance_role[-1], "ip": node["ip"], "port": replicaset_info["port"], "domain": node["domain"], @@ -475,7 +477,7 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: else: info["storages"].append( { - "role": instance_role[index], + "role": self.instance_role[index], "ip": node["ip"], "port": replicaset_info["port"], "domain": node["domain"], @@ -502,11 +504,15 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: for index, node in enumerate(self.payload["config"]["nodes"]): if index == len(self.payload["config"]["nodes"]) - 1: config["nodes"].append( - {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[-1]} + {"ip": node["ip"], "port": self.payload["config"]["port"], "role": self.instance_role[-1]} ) else: config["nodes"].append( - {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[index]} + { + "ip": node["ip"], + "port": self.payload["config"]["port"], + "role": self.instance_role[index], + } ) info["configs"].append(config) @@ -521,11 +527,11 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: for index, node in enumerate(shard["nodes"]): if index == len(shard["nodes"]) - 1: storage["nodes"].append( - {"role": instance_role[-1], "ip": node["ip"], "port": shard["port"]} + {"role": self.instance_role[-1], "ip": node["ip"], "port": shard["port"]} ) else: storage["nodes"].append( - {"role": instance_role[index], "ip": node["ip"], "port": shard["port"]} + {"role": self.instance_role[index], "ip": node["ip"], "port": shard["port"]} ) info["storages"].append(storage) return info @@ -541,11 +547,7 @@ def get_add_domain_to_dns_kwargs(self, cluster: bool) -> dict: } for node in self.replicaset_info["nodes"] ] - return { - "bk_biz_id": self.payload["bk_biz_id"], - "bk_cloud_id": self.replicaset_info["nodes"][0]["bk_cloud_id"], - "domains": domains, - } + bk_cloud_id = self.replicaset_info["nodes"][0]["bk_cloud_id"] else: domains = [ { @@ -554,11 +556,12 @@ def get_add_domain_to_dns_kwargs(self, cluster: bool) -> dict: } for node in self.payload["mongos"]["nodes"] ] - return { - "bk_biz_id": self.payload["bk_biz_id"], - "bk_cloud_id": self.payload["mongos"]["nodes"][0]["bk_cloud_id"], - "domains": domains, - } + bk_cloud_id = self.payload["mongos"]["nodes"][0]["bk_cloud_id"] + return { + "bk_biz_id": self.payload["bk_biz_id"], + "bk_cloud_id": bk_cloud_id, + "domains": domains, + } def get_add_shard_to_cluster_kwargs(self) -> dict: """把shard添加到cluster的kwargs""" @@ -1293,7 +1296,7 @@ def calc_scale(self, info: dict) -> dict: # 获取副本集老实例信息 cluster_id = info["cluster_id"] - self.get_cluster_info_deinstall(cluster_id=info["cluster_id"]) + self.get_cluster_info_deinstall(cluster_id=cluster_id) if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: # 获取副本集新机器的顺序 副本集容量变更独占机器 mongodb_host_order_by_tolerance = machine_order_by_tolerance( @@ -1415,6 +1418,275 @@ def get_host_scale(self, mongodb_type: str, info: dict): self.db_main_version = self.payload["db_version"].split(".")[0] self.payload["hosts"] = hosts + def get_host_increase_node(self, info: dict): + """cluster增加node获取主机""" + + hosts = [] + for host in info["add_shard_nodes"]: + hosts.append({"ip": host["ip"], "bk_cloud_id": host["bk_cloud_id"]}) + # db版本 + self.payload["db_version"] = info["db_version"] + self.db_main_version = self.payload["db_version"].split(".")[0] + self.payload["hosts"] = hosts + + def calc_increase_node(self, info: dict): + """增加节点计算cluster对应关系""" + + # 以IP为维度获取cluster对应关系 + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + if len(info["cluster_ids"]) > 1: + reuse_machine = True + else: + reuse_machine = False + add_shard_node = info["add_shard_node"] + index = info["add_shard_node"]["node_index"] + current_shard_nodes_num = info["current_shard_nodes_num"] + self.payload["replicaset_set"] = [] + self.payload["scale_outs"] = [] + for cluster_id in info["cluster_ids"]: + self.get_cluster_info_deinstall(cluster_id=cluster_id) + port = self.payload["nodes"][0]["port"] + role = self.instance_role[current_shard_nodes_num - 1 + index] + node_name = role.split("_")[1] + ip = add_shard_node["ip"] + bk_cloud_id = add_shard_node["bk_cloud_id"] + domain = "{}.{}.{}.db".format(node_name, self.payload["cluster_name"], self.payload["app"]) + db_instance = { + "cluster_id": cluster_id, + "role": role, + "ip": ip, + "port": port, + "bk_cloud_id": bk_cloud_id, + "domain": domain, + } + scale_out_instance = { + "cluster_id": cluster_id, + "shard": "", + "role": role, + "ip": ip, + "port": port, + "domain": domain, + "spec_id": info["resource_spec"]["shard_nodes"]["spec_id"], + "sepc_config": info["resource_spec"]["shard_nodes"], + "reuse_machine": reuse_machine, + } + self.payload["replicaset_set"].append(db_instance) + self.payload["scale_outs"].append(scale_out_instance) + elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + cluster_id = info["cluster_id"] + self.get_cluster_info_deinstall(cluster_id=info["cluster_id"]) + self.payload["shards_instance_relationships_by_ip"] = {} + self.payload["scale_out_instances_by_ip"] = {} + # 每台机器部署的实例数 + node_replica_count = info["node_replica_count"] + if node_replica_count > 1: + reuse_machine = True + else: + reuse_machine = False + # cluster的shard每增加一个节点所需要的机器数 + host_num_one_node = int(info["shards_num"] / node_replica_count) + # 分配机器，每增加一个node取一次对应的机器 + for node in range(info["add_shard_nodes_num"]): + for index, host in enumerate( + info["add_shard_nodes"][node * host_num_one_node : (node + 1) * host_num_one_node] + ): + # 把机器分配对应的shard + shards = self.payload["shards_nodes"][ + index * node_replica_count : (index + 1) * node_replica_count + ] + db_instances_by_ip = [] + scale_out_instances_by_ip = [] + for shard in shards: + # 单个shard的实例对应关系 + current_node_num = len(shard["nodes"]) + role = self.instance_role[current_node_num - 1 + node] + ip = host["ip"] + port = shard["nodes"][0]["port"] + db_instances_by_ip.append( + { + "cluster_id": cluster_id, + "ip": ip, + "bk_cloud_id": host["bk_cloud_id"], + "role": role, + "cluster_name": self.payload["cluster_name"], + "seg_range": shard["shard"], + "db_version": info["db_version"], + "port": shard["nodes"][0]["port"], + } + ) + scale_out_instances_by_ip.append( + { + "cluster_id": cluster_id, + "shard": shard["shard"], + "role": role, + "ip": ip, + "port": port, + "domain": "", + "spec_id": info["resource_spec"]["shard_nodes"]["spec_id"], + "sepc_config": info["resource_spec"]["shard_nodes"], + "reuse_machine": reuse_machine, + } + ) + self.payload["shards_instance_relationships_by_ip"][host["ip"]] = db_instances_by_ip + self.payload["scale_out_instances_by_ip"][host["ip"]] = scale_out_instances_by_ip + + def get_increase_node_kwargs(self, info: dict) -> dict: + """添加node的kwargs""" + + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": info["exec_bk_cloud_id"], + "exec_ip": info["exec_ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.MongoDReplace, + "file_path": self.file_path, + "payload": { + "ip": info["ip"], + "port": info["port"], + "sourceIP": "", + "sourcePort": 0, + "sourceDown": False, + "adminUsername": info["admin_user"], + "adminPassword": info["admin_password"], + "targetIP": info["target"]["ip"], + "targetPort": info["target"]["port"], + "targetPriority": "1", + "targetHidden": "0", + }, + }, + } + + def get_meta_scale_node_kwargs(self, info: list, increase: bool) -> dict: + """增减shard节点数修改meta的""" + + if increase: + scale_out = info + scale_in = [] + else: + scale_out = [] + scale_in = info + return { + "created_by": self.payload["created_by"], + "immute_domain": "", + "cluster_id": info[0]["cluster_id"], + "bk_biz_id": self.payload["bk_biz_id"], + "scale_out": scale_out, + "scale_in": scale_in, + } + + def calc_reduce_node(self, info: dict): + """减少节点计算cluster对应关系""" + + # 获取副本集老实例信息 + reduce_shard_nodes = info["reduce_shard_nodes"] + hosts_set = set() + hosts = [] + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + db_instances = [] + for cluster_id in info["cluster_ids"]: + self.get_cluster_info_deinstall(cluster_id=cluster_id) + current_node_num = len(self.payload["nodes"]) + for index in range(reduce_shard_nodes): + role = self.instance_role[current_node_num - 2 - index] + for node in self.payload["nodes"]: + if node["instance_role"] == role: + db_instances.append( + { + "cluster_id": cluster_id, + "role": role, + "cluster_name": self.payload["cluster_name"], + "ip": node["ip"], + "port": node["port"], + "bk_cloud_id": node["bk_cloud_id"], + "domain": node["domain"], + } + ) + hosts_set.add(node["ip"]) + break + self.payload["db_instances"] = db_instances + bk_cloud_id = self.payload["db_instances"][0]["bk_cloud_id"] + + elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + cluster_id = info["cluster_id"] + self.get_cluster_info_deinstall(cluster_id=cluster_id) + self.payload["shards_instance_relationships"] = {} + bk_cloud_id = "" + # 所有shard的实例对应关系 + shards_instance_relationships = {} + for shard in self.payload["shards_nodes"]: + shards_instance_relationships[shard["shard"]] = [] + for shard in self.payload["shards_nodes"]: + current_node_num = len(shard["nodes"]) + for index in range(reduce_shard_nodes): + role = self.instance_role[current_node_num - 2 - index] + for node in shard["nodes"]: + if node["instance_role"] == role: + shards_instance_relationships[shard["shard"]].append( + { + "cluster_id": cluster_id, + "ip": node["ip"], + "bk_cloud_id": node["bk_cloud_id"], + "role": role, + "cluster_name": self.payload["cluster_name"], + "seg_range": shard["shard"], + "port": node["port"], + } + ) + bk_cloud_id = node["bk_cloud_id"] + hosts_set.add(node["ip"]) + self.payload["shards_instance_relationships"] = shards_instance_relationships + for host in hosts_set: + hosts.append({"ip": host, "bk_cloud_id": bk_cloud_id}) + self.payload["hosts"] = hosts + + # def get_host_reduce_node(self): + # """cluster减少node获取主机""" + # + # hosts_set = set() + # hosts = [] + # if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + # bk_cloud_id = self.payload["db_instances"][0]["bk_cloud_id"] + # for host in self.payload["db_instances"]: + # hosts_set.add(host["ip"]) + # for host in hosts_set: + # hosts.append({"ip": host, "bk_cloud_id": bk_cloud_id}) + # elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + # for shard in self.payload["shards_instance_relationships"].values(): + # for node in shard: + # hosts_set.add(node["ip"]) + # + # + # + # self.payload["hosts"] = hosts + + def get_reduce_node_kwargs(self, info: dict) -> dict: + """减少node的kwargs""" + + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": info["exec_bk_cloud_id"], + "exec_ip": info["exec_ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.MongoDReplace, + "file_path": self.file_path, + "payload": { + "ip": info["ip"], + "port": info["port"], + "sourceIP": info["source"]["ip"], + "sourcePort": info["source"]["port"], + "sourceDown": False, + "adminUsername": info["admin_user"], + "adminPassword": info["admin_password"], + "targetIP": "", + "targetPort": 0, + "targetPriority": "", + "targetHidden": "", + }, + }, + } + @dataclass() class CommonContext: diff --git a/dbm-ui/backend/flow/views/mongodb_scene.py b/dbm-ui/backend/flow/views/mongodb_scene.py index e6a612393e..6191ad1c80 100644 --- a/dbm-ui/backend/flow/views/mongodb_scene.py +++ b/dbm-ui/backend/flow/views/mongodb_scene.py @@ -166,7 +166,7 @@ def post(request): return Response({"root_id": root_id}) -class MongoDBDeInstallSView(FlowTestView): +class MongoDBDeInstallView(FlowTestView): """ mongodb卸载 """ @@ -178,7 +178,7 @@ def post(request): return Response({"root_id": root_id}) -class MongoDBScaleSView(FlowTestView): +class MongoDBScaleView(FlowTestView): """ mongodb容量变更 """ @@ -188,3 +188,27 @@ def post(request): root_id = uuid.uuid1().hex MongoDBController(root_id=root_id, ticket_data=request.data).scale_cluster() return Response({"root_id": root_id}) + + +class MongoDBIncreaseNodeView(FlowTestView): + """ + mongodb增加node + """ + + @staticmethod + def post(request): + root_id = uuid.uuid1().hex + MongoDBController(root_id=root_id, ticket_data=request.data).increase_node() + return Response({"root_id": root_id}) + + +class MongoDBReduceNodeView(FlowTestView): + """ + mongodb减少node + """ + + @staticmethod + def post(request): + root_id = uuid.uuid1().hex + MongoDBController(root_id=root_id, ticket_data=request.data).reduce_node() + return Response({"root_id": root_id})