From 3ccf47e00fc0ed622a50950a3057143d8922acaa Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Wed, 20 Nov 2024 20:18:52 +0800 Subject: [PATCH] fix: bugs --- dbm-ui/backend/db_dirty/handlers.py | 176 +----------------- dbm-ui/backend/db_dirty/models.py | 21 ++- ..._20241014_1042_0043_auto_20241015_2128.py} | 2 +- 3 files changed, 14 insertions(+), 185 deletions(-) rename dbm-ui/backend/db_meta/migrations/{0044_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py => 0045_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py} (82%) diff --git a/dbm-ui/backend/db_dirty/handlers.py b/dbm-ui/backend/db_dirty/handlers.py index fc2004179a..0460d4ac52 100644 --- a/dbm-ui/backend/db_dirty/handlers.py +++ b/dbm-ui/backend/db_dirty/handlers.py @@ -10,28 +10,18 @@ """ import itertools import logging -from collections import defaultdict -from typing import Any, Dict, List +from typing import List from django.utils.translation import ugettext as _ -from backend import env -from backend.components import CCApi -from backend.configuration.constants import SystemSettingsEnum -from backend.configuration.models import SystemSettings from backend.db_dirty.constants import MachineEventType, PoolType from backend.db_dirty.exceptions import PoolTransferException from backend.db_dirty.models import DirtyMachine, MachineEvent -from backend.db_meta.models import AppCache -from backend.db_services.ipchooser.constants import IDLE_HOST_MODULE -from backend.db_services.ipchooser.handlers.topo_handler import TopoHandler -from backend.db_services.ipchooser.query.resource import ResourceQueryHelper from backend.flow.consts import FAILED_STATES from backend.flow.utils.cc_manage import CcManage from backend.ticket.builders import BuilderFactory from backend.ticket.builders.common.base import fetch_apply_hosts from backend.ticket.models import Flow, Ticket -from backend.utils.batch_request import request_multi_thread logger = logging.getLogger("root") @@ -66,170 +56,6 @@ def transfer_hosts_to_pool(cls, operator: str, bk_host_ids: List[int], source: P else: raise PoolTransferException(_("{}--->{}转移不合法").format(source, target)) - @classmethod - def query_dirty_machine_records(cls, bk_host_ids: List[int]): - """ - 查询污点池主机信息 TODO: 污点池废弃，代码将被移除 - @param bk_host_ids: 主机列表 - """ - - def get_module_data(data): - params, res = data - params = params["params"] - return [{"bk_biz_id": params["bk_biz_id"], **d} for d in res] - - if not bk_host_ids: - return [] - - # 如果传入的列表已经是DirtyMachine，则直接用 - if not isinstance(bk_host_ids[0], DirtyMachine): - dirty_machines = DirtyMachine.objects.filter(bk_host_id__in=bk_host_ids) - else: - dirty_machines = bk_host_ids - bk_host_ids = [dirty.bk_host_id for dirty in dirty_machines] - - # 缓存云区域和业务信息 - bk_biz_ids = [dirty_machine.bk_biz_id for dirty_machine in dirty_machines] - for_biz_infos = AppCache.batch_get_app_attr(bk_biz_ids=bk_biz_ids, attr_name="bk_biz_name") - cloud_info = ResourceQueryHelper.search_cc_cloud(get_cache=True) - - # 查询污点主机当前所处的模块 - host_topo_infos = CCApi.find_host_biz_relations(params={"bk_host_id": bk_host_ids}) - host__topo_info_map: Dict[int, List] = defaultdict(list) - biz__modules_map: Dict[int, List] = defaultdict(list) - for topo in host_topo_infos: - host__topo_info_map[topo["bk_host_id"]].append(topo) - biz__modules_map[topo["bk_biz_id"]].append(topo["bk_module_id"]) - # 批量获取业务下模块信息 - module_infos = request_multi_thread( - func=CCApi.find_module_batch, - params_list=[ - { - "params": {"bk_biz_id": biz, "bk_ids": modules, "fields": ["bk_module_id", "bk_module_name"]}, - "use_admin": True, - } - for biz, modules in biz__modules_map.items() - ], - get_data=get_module_data, - in_order=True, - ) - module_infos = list(itertools.chain(*module_infos)) - biz__module__module_name: Dict[int, Dict[int, str]] = defaultdict(dict) - for info in module_infos: - biz__module__module_name[info["bk_biz_id"]][info["bk_module_id"]] = info["bk_module_name"] - - # 获取污点池模块 - system_manage_topo = SystemSettings.get_setting_value(key=SystemSettingsEnum.MANAGE_TOPO.value) - dirty_module = system_manage_topo["dirty_module_id"] - - # 获取污点池列表信息 - dirty_machine_list: List[Dict] = [] - for dirty in dirty_machines: - # 填充污点池主机基础信息 - dirty_machine_info = { - "ip": dirty.ip, - "bk_host_id": dirty.bk_host_id, - "bk_cloud_name": cloud_info[str(dirty.bk_cloud_id)]["bk_cloud_name"], - "bk_cloud_id": dirty.bk_cloud_id, - "bk_biz_name": for_biz_infos[int(dirty.bk_biz_id)], - "bk_biz_id": dirty.bk_biz_id, - "ticket_type": dirty.ticket.ticket_type, - "ticket_id": dirty.ticket.id, - "ticket_type_display": dirty.ticket.get_ticket_type_display(), - "task_id": dirty.flow.flow_obj_id, - "operator": dirty.ticket.creator, - "is_dirty": True, - } - - # 如果主机已经不存在于cc，则仅能删除记录 - if dirty.bk_host_id not in host__topo_info_map: - dirty_machine_info.update(is_dirty=False) - dirty_machine_list.append(dirty_machine_info) - continue - - # 补充主机所在的模块信息 - host_in_module = [ - { - "bk_module_id": h["bk_module_id"], - "bk_module_name": biz__module__module_name[h["bk_biz_id"]].get(h["bk_module_id"], ""), - } - for h in host__topo_info_map[dirty.bk_host_id] - ] - dirty_machine_info.update(bk_module_infos=host_in_module) - - # 如果主机 不处于/不仅仅处于【污点池】中，则不允许移入待回收 - host = host__topo_info_map[dirty.bk_host_id][0] - if len(host__topo_info_map[dirty.bk_host_id]) > 1: - dirty_machine_info.update(is_dirty=False) - elif host["bk_biz_id"] != env.DBA_APP_BK_BIZ_ID or host["bk_module_id"] != dirty_module: - dirty_machine_info.update(is_dirty=False) - - dirty_machine_list.append(dirty_machine_info) - - dirty_machine_list.sort(key=lambda x: x["ticket_id"], reverse=True) - return dirty_machine_list - - @classmethod - def insert_dirty_machines(cls, bk_biz_id: int, bk_host_ids: List[Dict[str, Any]], ticket: Ticket, flow: Flow): - """ - 将机器导入到污点池中 TODO: 污点池废弃，代码将被移除 - @param bk_biz_id: 业务ID - @param bk_host_ids: 主机列表 - @param ticket: 关联的单据 - @param flow: 关联的flow任务 - """ - # 查询污点机器信息 - host_property_filter = { - "condition": "AND", - "rules": [{"field": "bk_host_id", "operator": "in", "value": bk_host_ids}], - } - dirty_host_infos = CCApi.list_hosts_without_biz( - { - # 默认一次性录入的机器不会超过500 - "page": {"start": 0, "limit": 500, "sort": "bk_host_id"}, - "host_property_filter": host_property_filter, - "fields": ["bk_host_id", "bk_cloud_id", "bk_host_innerip"], - }, - use_admin=True, - )["info"] - - # 获取业务空闲机模块，资源池模块和污点池模块 - idle_module = CcManage(bk_biz_id, "").get_biz_internal_module(bk_biz_id)[IDLE_HOST_MODULE]["bk_module_id"] - system_manage_topo = SystemSettings.get_setting_value(key=SystemSettingsEnum.MANAGE_TOPO.value) - resource_module, dirty_module = system_manage_topo["resource_module_id"], system_manage_topo["dirty_module_id"] - # 获取主机的拓扑信息(注：这里不能带上业务信息，因为主机可能转移业务) - host_topo_infos = TopoHandler.query_host_set_module(bk_host_ids=bk_host_ids)["hosts_topo_info"] - # 将污点机器信息转移至DBA污点池模(如果污点机器不在空闲机/资源池，则放弃转移，认为已到正确拓扑) - transfer_host_ids = [ - info["bk_host_id"] - for info in host_topo_infos - if not set(info["bk_module_ids"]) - {resource_module, idle_module} - ] - if transfer_host_ids: - update_host_properties = {"dbm_meta": [], "need_monitor": False, "update_operator": False} - CcManage(bk_biz_id=env.DBA_APP_BK_BIZ_ID, cluster_type="").transfer_host_module( - transfer_host_ids, target_module_ids=[dirty_module], update_host_properties=update_host_properties - ) - - # 录入污点池表中 - exist_dirty_machine_ids = list( - DirtyMachine.objects.filter(bk_host_id__in=bk_host_ids).values_list("bk_host_id", flat=True) - ) - DirtyMachine.objects.bulk_create( - [ - DirtyMachine( - ticket=ticket, - flow=flow, - ip=host["bk_host_innerip"], - bk_biz_id=bk_biz_id, - bk_host_id=host["bk_host_id"], - bk_cloud_id=host["bk_cloud_id"], - ) - for host in dirty_host_infos - if host["bk_host_id"] not in exist_dirty_machine_ids - ] - ) - @classmethod def handle_dirty_machine(cls, ticket_id, root_id, origin_tree_status, target_tree_status): """处理执行失败/重试成功涉及的污点池机器""" diff --git a/dbm-ui/backend/db_dirty/models.py b/dbm-ui/backend/db_dirty/models.py index f083086e6e..aded49753e 100644 --- a/dbm-ui/backend/db_dirty/models.py +++ b/dbm-ui/backend/db_dirty/models.py @@ -59,15 +59,18 @@ def hosts_pool_transfer(cls, bk_biz_id, hosts, pool, operator="", ticket=None): host_ids = [host["bk_host_id"] for host in hosts] # 主机转入污点/故障池，说明第一次被纳管到池 - if pool in [PoolType.Fault, PoolType.Dirty]: - hosts_pool = [ - cls(bk_biz_id=bk_biz_id, pool=pool, ticket=ticket, creator=operator, updater=operator, **host) - for host in hosts - ] - cls.objects.bulk_create(hosts_pool) - # 待回收只会从故障池转移 - elif pool == PoolType.Recycle: - cls.objects.filter(bk_host_id__in=host_ids).update(pool=pool, ticket=ticket) + # 待回收会从故障池、资源池转移 + # 因此这里判断主机不存在就创建，否则更新 + if pool in [PoolType.Fault, PoolType.Dirty, PoolType.Recycle]: + handle_hosts = cls.objects.filter(bk_host_id__in=host_ids) + if handle_hosts.count() == len(host_ids): + handle_hosts.update(pool=pool, ticket=ticket) + else: + handle_hosts = [ + cls(bk_biz_id=bk_biz_id, pool=pool, ticket=ticket, creator=operator, updater=operator, **host) + for host in hosts + ] + cls.objects.bulk_create(handle_hosts) # 回收机器只能从待回收转移，删除池纳管记录 # 重导入回资源池，删除池纳管记录 elif pool in [PoolType.Recycled, PoolType.Resource]: diff --git a/dbm-ui/backend/db_meta/migrations/0044_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py b/dbm-ui/backend/db_meta/migrations/0045_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py similarity index 82% rename from dbm-ui/backend/db_meta/migrations/0044_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py rename to dbm-ui/backend/db_meta/migrations/0045_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py index c6fe2d2ee4..b341e2d078 100644 --- a/dbm-ui/backend/db_meta/migrations/0044_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py +++ b/dbm-ui/backend/db_meta/migrations/0045_merge_0043_auto_20241014_1042_0043_auto_20241015_2128.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("db_meta", "0043_auto_20241014_1042"), + ("db_meta", "0044_deviceclass"), ("db_meta", "0043_auto_20241015_2128"), ]