fix: mongodb单据协议调整

dbm-ui/backend/bk_web/middleware.py

@@ -88,6 +88,7 @@ def process_view(self, request, view, args, kwargs):
         2. 如果用的dbm的APP CODE和APP TOKEN，则认为是服务内调用，授予超级用户
         3. 如果是apigw认证通过，则授予请求头中X-Bkapi-Apigw的用户
         """
+        setattr(request, "_dont_enforce_csrf_checks", True)
 
         def authorize_admin_user():
             request.user = User(username="admin", is_superuser=True)

dbm-ui/backend/db_dirty/models.py

@@ -116,7 +116,7 @@ def host_event_trigger(cls, bk_biz_id, hosts, event, operator="", ticket=None, s
         pool = MACHINE_EVENT__POOL_MAP.get(event)
         # 如果主机非标准话，则查询cc
         if not standard:
-            hosts = ResourceHandler.standardized_resource_host(hosts, bk_biz_id)
+            hosts = ResourceHandler.standardized_resource_host(hosts)
         # 主机池流转
         if pool:
             DirtyMachine.hosts_pool_transfer(bk_biz_id, hosts, pool, operator, ticket)

dbm-ui/backend/db_services/dbresource/handlers.py

@@ -15,6 +15,7 @@
 from django.forms import model_to_dict
 from django.utils.translation import ugettext as _
 
+from backend.components import CCApi
 from backend.components.dbresource.client import DBResourceApi
 from backend.components.gse.client import GseApi
 from backend.db_meta.enums.spec import SpecClusterType, SpecMachineType
@@ -469,10 +470,15 @@ def _format_resource_fields(data, _cloud_info, _biz_infos, _tag_infos):
     def standardized_resource_host(cls, hosts, bk_biz_id=None):
         """标准化主机信息，将cc字段统一成资源池字段"""
         host_ids = [host["bk_host_id"] for host in hosts]
+        # 获取主机通用信息
         hosts = ResourceQueryHelper.search_cc_hosts(role_host_ids=host_ids)
+        # 获取主机拓扑信息
+        host_topos = CCApi.find_host_biz_relations({"bk_host_id": host_ids})
+        host_biz_map = [{host["bk_host_id"]: host["bk_biz_id"]} for host in host_topos]
+        # 补充主机信息
         for host in hosts:
             host.update(
-                bk_biz_id=bk_biz_id,
+                bk_biz_id=host_biz_map[host["bk_host_id"]],
                 ip=host.get("bk_host_innerip"),
                 city=host.get("idc_city_name"),
                 host_id=host.get("bk_host_id"),

dbm-ui/backend/ticket/builders/__init__.py

@@ -21,7 +21,7 @@
 from backend import env
 from backend.components.dbresource.client import DBResourceApi
 from backend.configuration.constants import DBType, SystemSettingsEnum
-from backend.configuration.models import DBAdministrator, SystemSettings
+from backend.configuration.models import BizSettings, DBAdministrator, SystemSettings
 from backend.db_dirty.constants import MachineEventType, PoolType
 from backend.db_dirty.models import DirtyMachine, MachineEvent
 from backend.db_meta.models import AppCache, Cluster
@@ -293,7 +293,6 @@ def format_ticket_data(self):
             "os_type": self.ticket_data["recycle_hosts"][0]["os_type"],
             "db_type": self.ticket.group,
         }
-        self.add_common_params()
 
     def post_callback(self):
         # 转移到故障池，记录机器事件(如果是资源池则资源导入后会记录)
@@ -319,6 +318,7 @@ def __init__(self, ticket: Ticket):
 
     def format_ticket_data(self):
         recycle_hosts = self.ticket_data["recycle_hosts"]
+        # 我们认为，在资源申请的情况下，不会混用多个集群类型
         self.ticket_data = {
             "ticket_id": self.ticket.id,
             "for_biz": self.ticket_data["ip_recycle"]["for_biz"],
@@ -328,8 +328,9 @@ def format_ticket_data(self):
             "operator": self.ticket.creator,
             # 标记为退回
             "return_resource": True,
+            # 要查询主机实际的业务管控
+            "bk_biz_id": recycle_hosts[0]["bk_host_id"],
         }
-        self.add_common_params()
 
     def pre_callback(self):
         # 在run的时候才会生成task id，此时要更新到资源池参数里面
@@ -583,8 +584,10 @@ def register(cls, ticket_type: str, **kwargs) -> Callable:
         @param ticket_type: 单据类型
         @param kwargs: 单据注册的额外信息，主要是将单据归为不同的集合中，目前有这几种类型
         1. is_apply: bool ---- 表示单据是否是部署类单据(类似集群的部署，扩容，替换等)
-        2. phase: ClusterPhase ---- 表示单据与集群状态的映射
+        2. is_recycle: bool ---- 表示单据是否是下架类单据(类似集群的下架，缩容，替换等)
+        3. phase: ClusterPhase ---- 表示单据与集群状态的映射
         4. action: ActionMeta ---- 表示单据与权限动作的映射
+        5. is_sensitive: bool --- 是否为敏感类单据（有特殊鉴权）
         """
 
         def inner_wrapper(wrapped_class: TicketFlowBuilder) -> TicketFlowBuilder:

dbm-ui/backend/ticket/builders/common/base.py

@@ -502,18 +502,16 @@ def patch_instance_details(self):
 
     def patch_recycle_host_details(self):
         """补充回收主机信息，在回收类单据一定调用此方法"""
-        bk_biz_id = self.ticket.bk_biz_id
         recycle_hosts = fetch_recycle_hosts(self.ticket.details)
         if not recycle_hosts:
             return
-        self.ticket.details["recycle_hosts"] = ResourceHandler.standardized_resource_host(recycle_hosts, bk_biz_id)
+        self.ticket.details["recycle_hosts"] = ResourceHandler.standardized_resource_host(recycle_hosts)
 
     def patch_recycle_cluster_details(self, role=None):
         """补充集群下架后回收主机信息，在下架类单据一定调用此方法"""
-        bk_biz_id = self.ticket.bk_biz_id
         recycle_hosts = Cluster.get_cluster_related_machines(fetch_cluster_ids(self.ticket.details), role)
         recycle_hosts = [{"bk_host_id": host.bk_host_id} for host in recycle_hosts]
-        self.ticket.details["recycle_hosts"] = ResourceHandler.standardized_resource_host(recycle_hosts, bk_biz_id)
+        self.ticket.details["recycle_hosts"] = ResourceHandler.standardized_resource_host(recycle_hosts)
 
     def patch_ticket_detail(self):
         if self.need_patch_cluster_details:

dbm-ui/backend/ticket/builders/mongodb/mongo_cutoff.py

@@ -9,6 +9,7 @@
 specific language governing permissions and limitations under the License.
 """
 import itertools
+from collections import defaultdict
 
 from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
@@ -20,6 +21,7 @@
 from backend.db_services.mongodb.resources.query import MongoDBListRetrieveResource
 from backend.flow.engine.controller.mongodb import MongoDBController
 from backend.ticket import builders
+from backend.ticket.builders.common.base import HostRecycleSerializer
 from backend.ticket.builders.mongodb.base import (
     BaseMongoDBOperateDetailSerializer,
     BaseMongoDBOperateResourceParamBuilder,
@@ -34,6 +36,7 @@ class ACutoffDetailSerializer(serializers.Serializer):
         class IpSpecSLZ(serializers.Serializer):
             ip = serializers.CharField(help_text=_("替换主机IP"))
             bk_cloud_id = serializers.IntegerField(help_text=_("主机所在云区域"))
+            bk_host_id = serializers.IntegerField(help_text=_("替换的主机ID"))
             spec_id = serializers.IntegerField(help_text=_("规格ID"))
 
         cluster_id = serializers.IntegerField(help_text=_("集群ID"))
@@ -45,6 +48,7 @@ class IpSpecSLZ(serializers.Serializer):
         help_text=_("主机来源"), choices=IpSource.get_choices(), default=IpSource.RESOURCE_POOL
     )
     infos = serializers.ListSerializer(help_text=_("整机替换信息"), child=ACutoffDetailSerializer())
+    ip_recycle = HostRecycleSerializer(help_text=_("主机回收信息"), default=HostRecycleSerializer.DEFAULT)
 
     def validate(self, attrs):
         # 校验替换的mongodb机器不在同一分片中
@@ -153,19 +157,21 @@ def post_callback(self):
             }
 
 
-@builders.BuilderFactory.register(TicketType.MONGODB_CUTOFF, is_apply=True)
+@builders.BuilderFactory.register(TicketType.MONGODB_CUTOFF, is_apply=True, is_recycle=True)
 class MongoDBCutoffApplyFlowBuilder(BaseMongoDBTicketFlowBuilder):
     serializer = MongoDBCutoffDetailSerializer
     inner_flow_builder = MongoDBCutoffFlowParamBuilder
     inner_flow_name = _("MongoDB 整机替换执行")
     resource_batch_apply_builder = MongoDBCutoffResourceParamBuilder
+    need_patch_recycle_host_details = True
 
-    def patch_ticket_resources(self):
+    def patch_ticket_resources_and_old_nodes(self):
         mongo_roles = [MachineType.MONGOS, MachineType.MONOG_CONFIG, MachineType.MONGODB]
         cluster_ids = [info["cluster_id"] for info in self.ticket.details["infos"]]
-        id__cluster = {cluster.id: cluster for cluster in Cluster.objects.filter(id__in=cluster_ids)}
+        cluster_map = Cluster.objects.in_bulk(cluster_ids)
+        old_nodes = defaultdict(list)
         for info in self.ticket.details["infos"]:
-            city = id__cluster[info["cluster_id"]].region
+            city = cluster_map[info["cluster_id"]].region
             # 打包资源信息：按照role_ip这样的命名格式构造每一个资源申请信息组，每组的城市同集群，数量为1
             resource_spec = {}
             for role in mongo_roles:
@@ -176,14 +182,16 @@ def patch_ticket_resources(self):
                         "count": 1,
                         "location_spec": {"city": city, "sub_zone_ids": []},
                     }
+                    old_nodes[role].append({"bk_host_id": host["bk_host_id"]})
             info["resource_spec"] = resource_spec
+            info["old_nodes"] = old_nodes
 
     def patch_ticket_specs(self):
         spec_ids = get_target_items_from_details(self.ticket.details["infos"], match_keys=["spec_id"])
         specs = {spec.spec_id: spec.get_spec_info() for spec in Spec.objects.filter(spec_id__in=spec_ids)}
         self.ticket.details["specs"] = specs
 
     def patch_ticket_detail(self):
-        self.patch_ticket_resources()
+        self.patch_ticket_resources_and_old_nodes()
         self.patch_ticket_specs()
         super().patch_ticket_detail()

dbm-ui/backend/ticket/builders/mongodb/mongo_destroy.py

@@ -15,12 +15,14 @@
 from backend.db_meta.models import AppCache
 from backend.flow.engine.controller.mongodb import MongoDBController
 from backend.ticket import builders
+from backend.ticket.builders.common.base import HostRecycleSerializer
 from backend.ticket.builders.mongodb.base import BaseMongoDBOperateDetailSerializer, BaseMongoDBTicketFlowBuilder
 from backend.ticket.constants import TicketType
 
 
 class MongoDBDestroyDetailSerializer(BaseMongoDBOperateDetailSerializer):
     cluster_ids = serializers.ListField(help_text=_("集群ID列表"), child=serializers.IntegerField())
+    ip_recycle = HostRecycleSerializer(help_text=_("主机回收信息"), default=HostRecycleSerializer.DEFAULT)
 
     def validate(self, attrs):
         return attrs
@@ -34,8 +36,9 @@ def format_ticket_data(self):
         self.ticket_data["bk_app_abbr"] = AppCache.objects.get(bk_biz_id=bk_biz_id).db_app_abbr
 
 
-@builders.BuilderFactory.register(TicketType.MONGODB_DESTROY, phase=ClusterPhase.DESTROY)
+@builders.BuilderFactory.register(TicketType.MONGODB_DESTROY, phase=ClusterPhase.DESTROY, is_recycle=True)
 class MongoDBDestroyApplyFlowBuilder(BaseMongoDBTicketFlowBuilder):
     serializer = MongoDBDestroyDetailSerializer
     inner_flow_builder = MongoDBDestroyFlowParamBuilder
     inner_flow_name = _("MongoDB 集群下架")
+    need_patch_recycle_cluster_details = True

dbm-ui/backend/ticket/builders/mongodb/mongo_reduce_mongos.py

@@ -15,7 +15,7 @@
 from backend.db_meta.models import AppCache, Cluster
 from backend.flow.engine.controller.mongodb import MongoDBController
 from backend.ticket import builders
-from backend.ticket.builders.common.base import CommonValidate, HostInfoSerializer
+from backend.ticket.builders.common.base import CommonValidate, HostInfoSerializer, HostRecycleSerializer
 from backend.ticket.builders.mongodb.base import (
     BaseMongoDBOperateDetailSerializer,
     BaseMongoOperateFlowParamBuilder,
@@ -26,35 +26,36 @@
 
 class MongoDBReduceMongosDetailSerializer(BaseMongoDBOperateDetailSerializer):
     class ReduceMongosDetailSerializer(serializers.Serializer):
+        class OldNodesSerializer(serializers.Serializer):
+            mongos = serializers.ListSerializer(help_text=_("缩容mongos"), child=HostInfoSerializer())
+
         cluster_id = serializers.IntegerField(help_text=_("集群ID"))
         role = serializers.CharField(help_text=_("接入层角色"), required=False, default=MachineType.MONGOS)
-        reduce_nodes = serializers.ListSerializer(help_text=_("缩容节点"), child=HostInfoSerializer())
+        old_nodes = OldNodesSerializer(help_text=_("缩容信息"))
 
     is_safe = serializers.BooleanField(help_text=_("是否做安全检测"), default=True, required=False)
     infos = serializers.ListSerializer(help_text=_("缩容接入层申请信息"), child=ReduceMongosDetailSerializer())
+    ip_recycle = HostRecycleSerializer(help_text=_("主机回收信息"), default=HostRecycleSerializer.DEFAULT)
 
     def validate(self, attrs):
         cluster_ids = [info["cluster_id"] for info in attrs["infos"]]
-        id__cluster = {
-            cluster.id: cluster
-            for cluster in Cluster.objects.prefetch_related("proxyinstance_set__machine").filter(id__in=cluster_ids)
-        }
+        cluster_map = Cluster.objects.prefetch_related("proxyinstance_set__machine").in_bulk(cluster_ids)
 
         # 校验集群类型合法性
         CommonValidate.validated_cluster_type(cluster_ids, ClusterType.MongoShardedCluster)
 
         for info in attrs["infos"]:
-            cluster = id__cluster[info["cluster_id"]]
+            cluster = cluster_map[info["cluster_id"]]
             mongos_count = cluster.proxyinstance_set.count()
-            info["reduce_count"] = len(info["reduce_nodes"])
+            info["reduce_count"] = len(info["old_nodes"]["mongos"])
 
             # 缩容后的整体mongos机器数量不能小于2
             if mongos_count - info["reduce_count"] < 2:
                 raise serializers.ValidationError(_("缩容后的整体mongos机器数量不能小于2"))
 
             # 缩容后的整体mongos需要满足集群亲和性，等后续支持指定count缩容后才校验
             machines = [s.machine for s in cluster.proxyinstance_set.all()]
-            shrink_ips = [node["ip"] for node in info["reduce_nodes"]]
+            shrink_ips = [node["ip"] for node in info["old_nodes"]["mongos"]]
             self.validate_shrink_ip_machine_affinity(cluster, machines, shrink_ips)
 
             # 缩容的mongos机器台数不能高于当前规格台数, 且不能为负数。TODO: 等支持指定规格数量缩容后，才需要这个校验
@@ -71,10 +72,13 @@ def format_ticket_data(self):
         bk_biz_id = self.ticket_data["bk_biz_id"]
         self.ticket_data["db_app_abbr"] = AppCache.objects.get(bk_biz_id=bk_biz_id).db_app_abbr
         self.ticket_data["infos"] = self.add_cluster_info(self.ticket_data["infos"])
+        for info in self.ticket_data["infos"]:
+            info["reduce_nodes"] = info.pop("old_nodes")["mongos"]
 
 
-@builders.BuilderFactory.register(TicketType.MONGODB_REDUCE_MONGOS)
+@builders.BuilderFactory.register(TicketType.MONGODB_REDUCE_MONGOS, is_recycle=True)
 class MongoDBAddMongosApplyFlowBuilder(BaseMongoShardedTicketFlowBuilder):
     serializer = MongoDBReduceMongosDetailSerializer
     inner_flow_builder = MongoDBReduceMongosFlowParamBuilder
     inner_flow_name = _("MongoDB 缩容接入层执行")
+    need_patch_recycle_host_details = True

dbm-ui/backend/ticket/builders/mongodb/mongo_reduce_shard_nodes.py

@@ -19,6 +19,7 @@
 from backend.db_meta.models import AppCache, Cluster
 from backend.flow.engine.controller.mongodb import MongoDBController
 from backend.ticket import builders
+from backend.ticket.builders.common.base import HostRecycleSerializer
 from backend.ticket.builders.mongodb.base import BaseMongoDBOperateDetailSerializer, BaseMongoDBTicketFlowBuilder
 from backend.ticket.constants import TicketType
 
@@ -30,6 +31,7 @@ class ReduceMongosDetailSerializer(serializers.Serializer):
 
     is_safe = serializers.BooleanField(help_text=_("是否做安全检测"))
     infos = serializers.ListSerializer(help_text=_("缩容shard节点数信息"), child=ReduceMongosDetailSerializer())
+    ip_recycle = HostRecycleSerializer(help_text=_("主机回收信息"), default=HostRecycleSerializer.DEFAULT)
 
     def validate(self, attrs):
         cluster_ids = list(itertools.chain(*[info["cluster_ids"] for info in attrs["infos"]]))

dbm-ui/backend/ticket/builders/mongodb/mongo_scale_updown.py

@@ -19,6 +19,7 @@
 from backend.db_services.dbbase.constants import IpSource
 from backend.flow.engine.controller.mongodb import MongoDBController
 from backend.ticket import builders
+from backend.ticket.builders.common.base import HostRecycleSerializer, fetch_cluster_ids
 from backend.ticket.builders.mongodb.base import (
     BaseMongoDBOperateDetailSerializer,
     BaseMongoDBOperateResourceParamBuilder,
@@ -40,6 +41,7 @@ class ScaleUpDownDetailSerializer(serializers.Serializer):
         help_text=_("主机来源"), choices=IpSource.get_choices(), default=IpSource.RESOURCE_POOL
     )
     infos = serializers.ListSerializer(help_text=_("集群容量变更申请信息"), child=ScaleUpDownDetailSerializer())
+    ip_recycle = HostRecycleSerializer(help_text=_("主机回收信息"), default=HostRecycleSerializer.DEFAULT)
 
     def validate(self, attrs):
         # 校验count = 机器组数 * 集群分片节点数
@@ -108,9 +110,27 @@ def post_callback(self):
             next_flow.details["ticket_data"]["infos"] = mongo_type__apply_infos
 
 
-@builders.BuilderFactory.register(TicketType.MONGODB_SCALE_UPDOWN, is_apply=True)
+@builders.BuilderFactory.register(TicketType.MONGODB_SCALE_UPDOWN, is_apply=True, is_recycle=True)
 class MongoDBScaleUpDownFlowBuilder(BaseMongoShardedTicketFlowBuilder):
     serializer = MongoDBScaleUpDownDetailSerializer
     inner_flow_builder = MongoDBScaleUpDownFlowParamBuilder
     inner_flow_name = _("MongoDB 集群容量变更执行")
     resource_batch_apply_builder = MongoDBScaleUpDownResourceParamBuilder
+    need_patch_recycle_host_details = True
+
+    def patch_old_shard_nodes(self):
+        # 获取所有下架的mongodb节点
+        cluster_ids = fetch_cluster_ids(self.ticket.details)
+        cluster_map = Cluster.objects.prefetch_related("storageinstance_set__machine").in_bulk(cluster_ids)
+        for info in self.ticket.details["infos"]:
+            cluster = cluster_map[info["cluster_id"]]
+            old_hosts = [
+                s.machine.bk_host_id
+                for s in cluster.storageinstance_set.all()
+                if s.machine.machine_type == MachineType.MONGODB
+            ]
+            info["old_nodes"] = {"old_shard": [{"bk_host_id": host} for host in set(old_hosts)]}
+
+    def patch_ticket_detail(self):
+        self.patch_old_shard_nodes()
+        super().patch_ticket_detail()