From 356513297501162f8b12383d9fdbfd7e72c6d26b Mon Sep 17 00:00:00 2001 From: jpyoung3 <809608046@qq.com> Date: Fri, 8 Nov 2024 18:14:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=89=E8=A3=85=E9=A2=84=E8=AE=BE?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E9=94=81=E5=AE=9A=E7=89=88=E6=9C=AC(closed?= =?UTF-8?q?=20#2482)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/components/collections/plugin.py | 6 +- apps/backend/subscription/steps/adapter.py | 68 +++++++++++++++++-- apps/node_man/models.py | 2 + 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/apps/backend/components/collections/plugin.py b/apps/backend/components/collections/plugin.py index d87312bd24..07d5869bb5 100644 --- a/apps/backend/components/collections/plugin.py +++ b/apps/backend/components/collections/plugin.py @@ -280,11 +280,12 @@ def _execute(self, data, parent_data, common_data: PluginCommonData): # target_host_objs 的长度通常为1或2，此处也不必担心时间复杂度问题 # 指定 target_host 主要用于远程采集的场景，常见于第三方插件，如拨测 for host in target_host_objs: + bk_biz_id = host.bk_biz_id bk_host_id = host.bk_host_id os_type = host.os_type.lower() cpu_arch = host.cpu_arch group_id = create_group_id(subscription, subscription_instance.instance_info) - package = self.get_package(subscription_instance, policy_step_adapter, os_type, cpu_arch) + package = self.get_package(subscription_instance, policy_step_adapter, os_type, cpu_arch, bk_biz_id) ap_config = self.get_ap_config(ap_id_obj_map, host) setup_path, pid_path, log_path, data_path = self.get_plugins_paths( package, plugin_name, ap_config, group_id, subscription @@ -340,10 +341,11 @@ def get_package( policy_step_adapter: PolicyStepAdapter, os_type: str, cpu_arch: str, + bk_biz_id: int, ) -> models.Packages: """获取插件包对象""" try: - return policy_step_adapter.get_matching_package_obj(os_type, cpu_arch) + return policy_step_adapter.get_matching_package_obj(os_type, cpu_arch, bk_biz_id) except errors.PackageNotExists as error: # 插件包不支持或不存在时，记录异常信息，此实例不参与后续流程 self.move_insts_to_failed([subscription_instance.id], str(error)) diff --git a/apps/backend/subscription/steps/adapter.py b/apps/backend/subscription/steps/adapter.py index 56e3975788..8c0f6e3b31 100644 --- a/apps/backend/subscription/steps/adapter.py +++ b/apps/backend/subscription/steps/adapter.py @@ -16,6 +16,7 @@ from django.db.models import Max, Subquery, Value from django.utils.translation import ugettext as _ +from packaging import version from rest_framework import exceptions, serializers from apps.backend.subscription import errors @@ -285,13 +286,17 @@ def format2policy_packages_new( self, plugin_id: int, plugin_name: str, plugin_version: str, config_templates: List[Dict[str, Any]] ) -> List[Dict[str, Any]]: latest_flag: str = "latest" - is_tag: bool = Tag.objects.filter( + stable_flag: str = "stable" + is_latest_tag: bool = Tag.objects.filter( target_id=plugin_id, name=latest_flag, target_type=TargetType.PLUGIN.value ).exists() + is_stable_tag: bool = Tag.objects.filter( + target_id=plugin_id, name=stable_flag, target_type=TargetType.PLUGIN.value + ).exists() - if plugin_version != latest_flag or is_tag: + if plugin_version not in (latest_flag, stable_flag) or is_latest_tag or is_stable_tag: # 如果 latest 是 tag，走取指定版本的逻辑 - packages = models.Packages.objects.filter(project=plugin_name, version=plugin_version) + packages = self.get_packages(plugin_name, plugin_version) else: max_pkg_ids: List[int] = self.max_ids_by_key( list(models.Packages.objects.filter(project=plugin_name).values("id", "os", "cpu_arch")) @@ -306,11 +311,11 @@ def format2policy_packages_new( os_cpu__config_templates_map = defaultdict(list) for template in config_templates: is_main_template = template["is_main"] - if template["version"] != latest_flag or is_tag: + if template["version"] not in (latest_flag, stable_flag) or is_latest_tag or is_stable_tag: plugin_version_set = {plugin_version, "*"} else: - latest_packages_version_set = set(packages.values_list("version", flat=True)) - plugin_version_set = latest_packages_version_set | {"*"} + tag_packages_version_set = set(packages.values_list("version", flat=True)) + plugin_version_set = tag_packages_version_set | {"*"} max_config_tmpl_ids: typing.List[int] = self.max_ids_by_key( list( @@ -444,9 +449,15 @@ def get_matching_step_params(self, os_type: str = None, cpu_arch: str = None, os return self.os_key_params_map.get(os_key) return self.os_key_params_map.get(self.get_os_key(os_type, cpu_arch), {}) - def get_matching_package_obj(self, os_type: str, cpu_arch: str) -> models.Packages: + def get_matching_package_obj(self, os_type: str, cpu_arch: str, bk_biz_id: int) -> models.Packages: try: package = self.os_key_pkg_map[self.get_os_key(os_type, cpu_arch)] + plugin_version_config = models.GlobalSettings.get_config( + models.GlobalSettings.KeyEnum.PLUGIN_VERSION_CONFIG.value + ) + if bk_biz_id in plugin_version_config: + os_cpu__biz_pkg_map = self.get_biz_version(package, plugin_version_config, bk_biz_id) + package = os_cpu__biz_pkg_map[self.get_os_key(os_type, cpu_arch)] except KeyError: msg = _("插件 [{name}] 不支持 系统:{os_type}-架构:{cpu_arch}-版本:{plugin_version}").format( name=self.plugin_name, @@ -468,3 +479,46 @@ def get_matching_package_obj(self, os_type: str, cpu_arch: str) -> models.Packag def get_matching_config_tmpl_objs(self, os_type: str, cpu_arch: str) -> List[models.PluginConfigTemplate]: return self.config_tmpl_obj_gby_os_key.get(self.get_os_key(os_type, cpu_arch), []) + + def get_packages(self, plugin_name: str, plugin_version: str, biz_version: str = None): + # 对于不存的版本取最大 id 的 package + all_packages = models.Packages.objects.filter(project=plugin_name).values("id", "os", "cpu_arch", "version") + version_packages = {pkg["id"]: pkg for pkg in all_packages if pkg["version"] == plugin_version} + package_ids = set(version_packages.keys()) + for os in constants.PLUGIN_OS_TUPLE: + for cpu_arch in constants.CPU_TUPLE: + if not any( + pkg["os"] == os and pkg["cpu_arch"] == cpu_arch + for pkg in all_packages + if pkg["version"] == plugin_version + ): + max_pkg_ids: List[int] = self.max_ids_by_key( + [pkg for pkg in all_packages if pkg["os"] == os and pkg["cpu_arch"] == cpu_arch] + ) + package_ids.update(max_pkg_ids) + packages = models.Packages.objects.filter(id__in=package_ids) + if biz_version: + packages = packages.filter( + id__in=[pkg["id"] for pkg in packages if version.Version(pkg.version) <= version.Version(biz_version)] + ) + return packages + + def get_biz_version(self, package: models.Packages, plugin_version_config: Dict[str, Dict], bk_biz_id: int): + biz_version_config = plugin_version_config[str(bk_biz_id)] + biz_version = next( + ( + biz_plugin_version + for biz_plugin_name, biz_plugin_version in biz_version_config.items() + if package.project == biz_plugin_name + ), + None, + ) + os_cpu__biz_pkg_map = {} + if biz_version and version.Version(package.version) > version.Version(biz_version): + packages = self.get_packages(package.project, biz_version, biz_version) + os_cpu__biz_pkg_map = {self.get_os_key(package.os, package.cpu_arch): package for package in packages} + if not os_cpu__biz_pkg_map: + raise errors.PluginValidationError( + msg="插件 [{name}-{versions}] 不存在".format(name=self.plugin_name, versions=biz_version) + ) + return os_cpu__biz_pkg_map diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 7c9a84ddb5..b6f51ec460 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -168,6 +168,8 @@ class KeyEnum(Enum): AUTO_SELECT_INSTALL_CHANNEL_ONLY_DIRECT_AREA = "AUTO_SELECT_INSTALL_CHANNEL_ONLY_DIRECT_AREA" # 安装通道ID与网段列表映射 INSTALL_CHANNEL_ID_NETWORK_SEGMENT = "INSTALL_CHANNEL_ID_NETWORK_SEGMENT" + # 业务最大插件版本 + PLUGIN_VERSION_CONFIG = "PLUGIN_VERSION_CONFIG" key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True) v_json = JSONField(_("值"))