From bc93ef51c6eab00197f6351a6c5a6dfc6c490588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fdcdding=28=E4=B8=81=E8=B6=85=E8=BE=BE=29?= <1151627903@qq.com> Date: Tue, 5 Mar 2024 16:59:12 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=89=E8=A3=85=20Agent=20=E5=B0=86?= =?UTF-8?q?=E4=B8=8A=E6=8A=A5=E7=9A=84=20CPU=20=E6=9E=B6=E6=9E=84=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E6=9B=B4=E6=96=B0=E5=88=B0=20CMDB=20(closed=20#1708)?= =?UTF-8?q?=20#=20Reviewed,=20transaction=20id:=203399?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../collections/agent_new/install.py | 35 +++++++++++-- .../collections/agent_new/test_install.py | 18 +++++++ apps/node_man/constants.py | 49 +++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/apps/backend/components/collections/agent_new/install.py b/apps/backend/components/collections/agent_new/install.py index 8052fe914..92df5b7a3 100644 --- a/apps/backend/components/collections/agent_new/install.py +++ b/apps/backend/components/collections/agent_new/install.py @@ -44,7 +44,7 @@ from apps.prometheus.helper import SetupObserve from apps.utils import concurrent, sync from apps.utils.exc import ExceptionHandler -from common.api import JobApi +from common.api import CCApi, JobApi from common.log import logger from pipeline.core.flow import Service, StaticIntervalGenerator @@ -822,11 +822,13 @@ def _schedule(self, data, parent_data, callback_data=None): left_scheduling_sub_inst_ids = [] cpu_arch__host_id_map = defaultdict(list) os_version__host_id_map = defaultdict(list) + cpu_arch__sub_inst_id_map = defaultdict(list) host_id__agent_id_map: Dict[int, str] = {} for result in results: # 对于未完成的实例，记录下来到下一次schedule中继续检查 if not result["is_finished"]: left_scheduling_sub_inst_ids.append(result["sub_inst_id"]) + cpu_arch__sub_inst_id_map[result["cpu_arch"]].append(result["sub_inst_id"]) # 按 CPU 架构对主机进行分组 bk_host_id = common_data.sub_inst_id__host_id_map.get(result["sub_inst_id"]) cpu_arch__host_id_map[result["cpu_arch"]].append(bk_host_id) @@ -839,10 +841,8 @@ def _schedule(self, data, parent_data, callback_data=None): os_version = result.get("os_version", "") if os_version is not None: os_version__host_id_map[os_version].append(bk_host_id) - # 批量更新CPU架构 - for cpu_arch, bk_host_ids in cpu_arch__host_id_map.items(): - if cpu_arch: - models.Host.objects.filter(bk_host_id__in=bk_host_ids).update(cpu_arch=cpu_arch) + # 批量更新CPU架构并且上报至CMDB + self.update_db_and_report_cpu_arch(cpu_arch__host_id_map, cpu_arch__sub_inst_id_map) # 批量更新主机操作系统版本号 for os_version, bk_host_ids in os_version__host_id_map.items(): @@ -873,3 +873,28 @@ def _schedule(self, data, parent_data, callback_data=None): self.move_insts_to_failed(left_scheduling_sub_inst_ids, _("安装超时")) self.finish_schedule() data.outputs.polling_time = polling_time + POLLING_INTERVAL + + def update_db_and_report_cpu_arch(self, cpu_arch__host_id_map: defaultdict, cpu_arch__sub_inst_id_map: defaultdict): + """ + :param cpu_arch__host_id_map: key -> cpu arch; value -> list of bk_host_id + :param cpu_arch__sub_inst_id_map: key -> cpu arch; value -> list of sub_inst_id + return: + """ + update_list: List[Dict[str, Any]] = [] + for cpu_arch, bk_host_ids in cpu_arch__host_id_map.items(): + if cpu_arch: + models.Host.objects.filter(bk_host_id__in=bk_host_ids).update(cpu_arch=cpu_arch) + for host_id in bk_host_ids: + update_params: Dict[str, Any] = { + "bk_host_id": host_id, + "properties": { + "bk_cpu_architecture": constants.CmdbCpuArchType.cpu_type__arch_map().get(cpu_arch), + "bk_os_bit": constants.OsBitType.cpu_type__os_bit_map().get(cpu_arch), + } + } + self.log_info( + sub_inst_ids=cpu_arch__sub_inst_id_map[cpu_arch], + log_content=_("更新 CMDB 主机信息:\n {params}").format(params=json.dumps(update_params, indent=2)), + ) + update_list.append(update_params) + CCApi.batch_update_host({"update": update_list}) diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index 877d05755..e3cfd4f09 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -51,6 +51,7 @@ class InstallBaseTestCase(utils.AgentServiceBaseTestCase): NODE_TYPE = constants.NodeType.AGENT DOWNLOAD_PATH = "/tmp/data/bkee/public/bknodeman/download" JOB_API_MOCK_PATH = "apps.backend.components.collections.agent_new.install.JobApi" + CMDB_API_MOCK_PATH = "apps.backend.components.collections.agent_new.install.CCApi" EXECUTE_CMD_MOCK_PATH = "apps.backend.components.collections.agent_new.install.execute_cmd" PUT_FILE_MOCK_PATH = "apps.backend.components.collections.agent_new.install.put_file" CUSTOM_DATAIPC_DIR = "/var/run/gse_test" @@ -70,6 +71,11 @@ def init_mock_clients(self): return_type=mock_data_utils.MockReturnType.RETURN_VALUE.value, return_obj={"job_instance_id": 1} ), ) + self.cmdb_mock_client = api_mkd.cmdb.utils.CCApiMockClient( + batch_update_host=mock_data_utils.MockReturn( + return_type=mock_data_utils.MockReturnType.RETURN_VALUE.value, return_obj={"message": "success"} + ), + ) def init_redis_data(self): # 初始化redis数据，用于schedule时读取解析 @@ -147,6 +153,7 @@ def update_common_inputs(self): def start_patch(self): mock.patch(self.JOB_API_MOCK_PATH, self.job_mock_client).start() + mock.patch(self.CMDB_API_MOCK_PATH, self.cmdb_mock_client).start() mock.patch(target=self.EXECUTE_CMD_MOCK_PATH, return_value="").start() mock.patch(target=self.PUT_FILE_MOCK_PATH, return_value="").start() base.get_asyncssh_connect_mock_patch().start() @@ -1031,3 +1038,14 @@ def _test_shell_solution(self, validate_encrypted_password: bool): run_cmd, ], ) + + +class ReportCpuArchTestCase(LinuxInstallTestCase): + def tearDown(self) -> None: + mock_call_obj = self.cmdb_mock_client.batch_update_host.call_args + if mock_call_obj: + call_args = mock_call_obj[0][0] + self.assertEqual(call_args["update"][0]["bk_host_id"], self.obj_factory.bk_host_ids[0]) + self.assertEqual(call_args["update"][0]["properties"]["bk_cpu_architecture"], "arm") + self.assertEqual(call_args["update"][0]["properties"]["bk_os_bit"], "arm-64bit") + super().tearDown() diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index aaeb6cd64..5fecdf8e1 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -503,6 +503,11 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: } CMDB_CPU_MAP = {"x86": CpuType.x86, "arm": CpuType.aarch64} +CMDB_CPU_ARCH = ("x86", "arm") +CPU_ARCH_CHOICES = tuple_choices(CMDB_CPU_ARCH) +CpuArchType = choices_to_namedtuple(CPU_ARCH_CHOICES) +CMDB_CPU_ARCH_MAP = {"x86": CpuArchType.x86, "x86_64": CpuArchType.x86, "aarch64": CpuArchType.arm} + PACKAGE_PATH_RE = re.compile( f"(?Pexternal_)?plugins_(?P({'|'.join(map(str, PLUGIN_OS_TUPLE))}))" f"_(?P({'|'.join(map(str, CPU_TUPLE))})?$)" @@ -1137,3 +1142,47 @@ class CommonExecutionSolutionStepType(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DEPENDENCIES: _("依赖文件"), cls.COMMANDS: _("命令")} + + +class CmdbCpuArchType(EnhanceEnum): + X86 = "x86" + X86_64 = "x86" + AARCH64 = "arm" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.X86: _("CPU架构:x86"), + cls.X86_64: _("CPU架构:x86_64"), + cls.AARCH64: _("CPU架构:arm") + } + + @classmethod + def cpu_type__arch_map(cls): + return { + CpuType.x86: cls.X86.value, + CpuType.x86_64: cls.X86_64.value, + CpuType.aarch64: cls.AARCH64.value + } + + +class OsBitType(EnhanceEnum): + BIT32 = "32-bit" + BIT64 = "64-bit" + ARM64 = "arm-64bit" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.BIT32: _("操作系统位数:32-bit"), + cls.BIT64: _("操作系统位数:64-bit"), + cls.ARM64: _("操作系统位数:arm-64bit") + } + + @classmethod + def cpu_type__os_bit_map(cls): + return { + CpuType.x86: cls.BIT32.value, + CpuType.x86_64: cls.BIT64.value, + CpuType.aarch64: cls.ARM64.value + }