From cb859e87b50d56f315431d7b9b6e1fe4a0fd5feb Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Thu, 31 Aug 2023 11:23:12 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feature:=20=E6=94=AF=E6=8C=81gse2.0?= =?UTF-8?q?=E5=8D=95=E7=8B=AC=E4=BD=9C=E4=B8=BA=E5=BC=80=E5=85=B3=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/default.py | 5 +- env.py | 2 +- pipeline_plugins/cmdb_ip_picker/query.py | 20 ++--- .../variables/collections/sites/open/cc.py | 30 ++++--- .../collections/sites/open/ip_filter_base.py | 84 ++++++++++++++++--- 5 files changed, 105 insertions(+), 36 deletions(-) diff --git a/config/default.py b/config/default.py index e87f800caa..aac4748ba0 100644 --- a/config/default.py +++ b/config/default.py @@ -155,7 +155,9 @@ ENABLE_IPV6 = env.ENABLE_IPV6 # paasv3 和 开启了ipv6 才会尝试加载 BK_API_URL_TMPL 这个变量 -if env.IS_PAAS_V3 or ENABLE_IPV6: +ENABLE_GSE_V2 = env.ENABLE_GSE_V2 + +if env.IS_PAAS_V3 or ENABLE_IPV6 or ENABLE_GSE_V2: BK_API_URL_TMPL = env.BK_APIGW_URL_TMPL if env.IS_PAAS_V3: @@ -672,6 +674,7 @@ def monitor_report_config(): from bk_monitor_report import MonitorReporter # noqa from bk_monitor_report.contrib.celery import MonitorReportStep # noqa + from blueapps.core.celery import celery_app # noqa reporter = MonitorReporter( diff --git a/env.py b/env.py index 3b89e971f1..f77124aabd 100644 --- a/env.py +++ b/env.py @@ -98,7 +98,7 @@ ENABLE_SWAGGER_UI = os.getenv("BKAPP_ENABLE_SWAGGER_UI", False) ENABLE_IPV6 = False if os.getenv("BKAPP_ENABLE_IPV6") is None else True - +ENABLE_GSE_V2 = int(os.getenv("BKAPP_ENABLE_GSE_V2", 0)) == 1 # 流程最高嵌套层数 TEMPLATE_MAX_RECURSIVE_NUMBER = int(os.getenv("BKAPP_TEMPLATE_MAX_RECURSIVE_NUMBER", 200)) diff --git a/pipeline_plugins/cmdb_ip_picker/query.py b/pipeline_plugins/cmdb_ip_picker/query.py index 27c2e7e07f..f08a732ead 100644 --- a/pipeline_plugins/cmdb_ip_picker/query.py +++ b/pipeline_plugins/cmdb_ip_picker/query.py @@ -16,24 +16,23 @@ import ujson as json from django.http import JsonResponse from django.utils.translation import ugettext_lazy as _ - -from api.utils.request import batch_request from iam.contrib.http import HTTP_AUTH_FORBIDDEN_CODE from iam.exceptions import RawAuthFailedException +from api.utils.request import batch_request from gcloud.conf import settings from gcloud.utils import cmdb -from gcloud.utils.ip import format_sundry_ip from gcloud.utils.handlers import handle_api_error +from gcloud.utils.ip import format_sundry_ip +from .constants import ERROR_CODES, NO_ERROR from .utils import ( get_cmdb_topo_tree, - get_objects_of_topo_tree, - get_modules_of_bk_obj, - get_modules_id, get_ges_agent_status_ipv6, + get_modules_id, + get_modules_of_bk_obj, + get_objects_of_topo_tree, ) -from .constants import NO_ERROR, ERROR_CODES logger = logging.getLogger("root") get_client_by_user = settings.ESB_GET_CLIENT_BY_USER @@ -63,8 +62,8 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id= @return: """ default_host_fields = ["bk_host_id", "bk_host_name", "bk_cloud_id", "bk_host_innerip"] - if settings.ENABLE_IPV6: - # IPV6环境下才会尝试去拿agent主机 + if settings.ENABLE_IPV6 or settings.ENABLE_GSE_V2: + # IPV6环境下或者开启了GSE 2.0 版本 default_host_fields.append("bk_agent_id") fields = set(default_host_fields + json.loads(request.GET.get("fields", "[]"))) client = get_client_by_user(request.user.username) @@ -130,7 +129,7 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id= data.append(host_detail) if "agent" in fields: - if settings.ENABLE_IPV6: + if settings.ENABLE_IPV6 or settings.ENABLE_GSE_V2: # 开启IPV6将会调用网关进行查询 bk_agent_id_list = [] for host in data: @@ -156,6 +155,7 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id= if not host["bk_host_innerip"]: # 如果既没有如果bk_agent_id，又没有ipv4地址，说明这个主机石台没有安装agent的ipv6主机，忽略，不再查询agent状态, 直接重置为未知 host["agent"] = -1 + continue bk_agent_id = "{}:{}".format(host["bk_cloud_id"], host["bk_host_innerip"]) host["agent"] = agent_id_status_map.get(bk_agent_id, -1) else: diff --git a/pipeline_plugins/variables/collections/sites/open/cc.py b/pipeline_plugins/variables/collections/sites/open/cc.py index 1cb182c205..b7c5f8ee09 100644 --- a/pipeline_plugins/variables/collections/sites/open/cc.py +++ b/pipeline_plugins/variables/collections/sites/open/cc.py @@ -225,7 +225,9 @@ def _self_explain(cls, **kwargs) -> List[FieldExplain]: FieldExplain(key="${KEY._module}", type=Type.LIST, description="集群下的模块信息列表，元素类型为字典，键为模块名，值为模块下的主机列"), FieldExplain(key="${KEY.flat__ip_list}", type=Type.STRING, description="本次操作创建的所有集群下的主机（去重后），用 ',' 连接"), FieldExplain( - key="${KEY.flat__verbose_ip_list}", type=Type.STRING, description="返回的是本次操作创建的所有集群下的主机（未去重），用 ',' 连接", + key="${KEY.flat__verbose_ip_list}", + type=Type.STRING, + description="返回的是本次操作创建的所有集群下的主机（未去重），用 ',' 连接", ), FieldExplain( key="${KEY.flat__verbose_ip_module_list}", @@ -304,7 +306,13 @@ def _handle_value_with_ipv4(username, bk_biz_id, bk_supplier_account, host_field if not ip_list: return [] - hosts_list = get_business_host(username, bk_biz_id, bk_supplier_account, host_fields, ip_list,) + hosts_list = get_business_host( + username, + bk_biz_id, + bk_supplier_account, + host_fields, + ip_list, + ) return hosts_list @staticmethod @@ -408,15 +416,7 @@ def get_value(self): origin_ip_list = get_plat_ip_by_regex(origin_ips) filter_data = {**self.value, **self.pipeline_data} - if not settings.ENABLE_IPV6: - # 进行gse agent状态过滤 - gse_agent_status_filter = GseAgentStatusIpFilter(origin_ip_list, filter_data) - match_result_ip = gse_agent_status_filter.get_match_ip() - if not ip_cloud: - return ip_separator.join(["{}".format(host["ip"]) for host in match_result_ip]) - - return ip_separator.join(["{}:{}".format(host["bk_cloud_id"], host["ip"]) for host in match_result_ip]) - else: + if settings.ENABLE_IPV6: gse_agent_status_ipv6_filter = GseAgentStatusIpV6Filter(origin_ips, filter_data) match_result_ip = gse_agent_status_ipv6_filter.get_match_ip() if not ip_cloud: @@ -430,3 +430,11 @@ def get_value(self): else: result.append("{}:{}".format(host["bk_cloud_id"], host["ip"])) return ip_separator.join(result) + else: + # 进行gse agent状态过滤 + gse_agent_status_filter = GseAgentStatusIpFilter(origin_ip_list, filter_data) + match_result_ip = gse_agent_status_filter.get_match_ip() + if not ip_cloud: + return ip_separator.join(["{}".format(host["ip"]) for host in match_result_ip]) + + return ip_separator.join(["{}:{}".format(host["bk_cloud_id"], host["ip"]) for host in match_result_ip]) diff --git a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py index 53c4f54910..9d129b2874 100644 --- a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py +++ b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py @@ -13,15 +13,17 @@ import logging from abc import ABCMeta, abstractmethod +from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from gcloud.constants import GseAgentStatus from gcloud.conf import settings as gcloud_settings +from gcloud.constants import GseAgentStatus from gcloud.core.models import Project from gcloud.exceptions import ApiRequestError +from gcloud.utils import cmdb from gcloud.utils.handlers import handle_api_error -from gcloud.utils.ip import extract_ip_from_ip_str, get_ip_by_regex_type, IpRegexType -from pipeline_plugins.base.utils.inject import supplier_id_for_project, supplier_account_for_business +from gcloud.utils.ip import IpRegexType, extract_ip_from_ip_str, get_ip_by_regex_type +from pipeline_plugins.base.utils.inject import supplier_account_for_business, supplier_id_for_project from pipeline_plugins.cmdb_ip_picker.utils import get_ges_agent_status_ipv6 from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6 @@ -40,18 +42,57 @@ def get_match_ip(self): class GseAgentStatusIpFilter(IpFilterBase): - def get_match_ip(self): + def match_ges_v2(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list): - origin_ip_list = self.origin_ip_list - gse_agent_status = self.data.get("gse_agent_status", "") - username = self.data["executor"] - project_id = self.data["project_id"] - project = Project.objects.get(id=project_id) - bk_biz_id = project.bk_biz_id if project.from_cmdb else "" - bk_supplier_id = supplier_id_for_project(project_id) - if not origin_ip_list: - return [] + fields = ["bk_host_id", "bk_cloud_id", "bk_host_innerip", "bk_agent_id"] + # 生成一个host列表 + origin_hosts = { + "{}:{}".format(origin_host["bk_cloud_id"], origin_host["ip"]): origin_host for origin_host in origin_ip_list + } + + ip_list = [host["ip"] for host in origin_ip_list] + # 先去查出来所有host的gse_agent_id + hosts = cmdb.get_business_host(username, bk_biz_id, bk_supplier_id, host_fields=fields, ip_list=ip_list) + + # 构造一个{bk_agent_id: bk_cloud_id:ip}的字典 + remote_hosts = {} + + for host in hosts: + remote_host_value = "{}:{}".format(host["bk_cloud_id"], host["bk_host_innerip"]) + if remote_host_value in origin_hosts.keys(): + bk_agent_id = host.get("bk_agent_id") + if not bk_agent_id: + # 没有agent_id 使用 云区域+ip 组成 agent_id + bk_agent_id = remote_host_value + remote_hosts[bk_agent_id] = remote_host_value + + # 去查询agent状态 + agent_map = get_ges_agent_status_ipv6(bk_agent_id_list=list(remote_hosts.keys())) + + agent_online_ip_list = [] # 在线的ip的列表 + agent_offline_ip_list = [] # 不在线的ip的列表 + match_ip = [] # 过滤失败将不返回任何i + for bk_agent_id, agent_code in agent_map.items(): + origin_host_key = remote_hosts.get(bk_agent_id) + if not origin_host_key: + continue + origin_host = origin_hosts.get(origin_host_key) + if not origin_host: + continue + + if agent_code == GseAgentStatus.ONlINE.value: + agent_online_ip_list.append(origin_host) + if agent_code == GseAgentStatus.OFFLINE.value: + agent_offline_ip_list.append(origin_host) + + if gse_agent_status == GseAgentStatus.ONlINE.value: + match_ip = agent_online_ip_list + if gse_agent_status == GseAgentStatus.OFFLINE.value: + match_ip = agent_offline_ip_list + return match_ip + + def match_gse_v1(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list): match_ip = origin_ip_list if gse_agent_status in [GseAgentStatus.ONlINE.value, GseAgentStatus.OFFLINE.value]: client = get_client_by_user(username) @@ -81,6 +122,23 @@ def get_match_ip(self): return match_ip + def get_match_ip(self): + + origin_ip_list = self.origin_ip_list + gse_agent_status = self.data.get("gse_agent_status", "") + username = self.data["executor"] + project_id = self.data["project_id"] + project = Project.objects.get(id=project_id) + bk_biz_id = project.bk_biz_id if project.from_cmdb else "" + bk_supplier_id = supplier_id_for_project(project_id) + if not origin_ip_list: + return [] + + if settings.ENABLE_GSE_V2: + return self.match_ges_v2(gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list) + else: + return self.match_gse_v1(gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list) + class GseAgentStatusIpV6Filter: def __init__(self, ip_str, data): From ad4c28ef69fc5b6e61305e49ca07c709af7beedc Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Thu, 31 Aug 2023 15:28:45 +0800 Subject: [PATCH 2/2] =?UTF-8?q?minor:=20=E4=BF=AE=E6=94=B9=E6=8B=BC?= =?UTF-8?q?=E5=86=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline_plugins/cmdb_ip_picker/query.py | 4 ++-- pipeline_plugins/cmdb_ip_picker/utils.py | 16 ++++++++++------ .../collections/sites/open/ip_filter_base.py | 6 +++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pipeline_plugins/cmdb_ip_picker/query.py b/pipeline_plugins/cmdb_ip_picker/query.py index f08a732ead..d77aa3f9ff 100644 --- a/pipeline_plugins/cmdb_ip_picker/query.py +++ b/pipeline_plugins/cmdb_ip_picker/query.py @@ -28,7 +28,7 @@ from .constants import ERROR_CODES, NO_ERROR from .utils import ( get_cmdb_topo_tree, - get_ges_agent_status_ipv6, + get_gse_agent_status_ipv6, get_modules_id, get_modules_of_bk_obj, get_objects_of_topo_tree, @@ -143,7 +143,7 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id= bk_agent_id_list.append(bk_agent_id) try: - agent_id_status_map = get_ges_agent_status_ipv6(bk_agent_id_list) + agent_id_status_map = get_gse_agent_status_ipv6(bk_agent_id_list) except Exception as e: result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": e} return JsonResponse(result) diff --git a/pipeline_plugins/cmdb_ip_picker/utils.py b/pipeline_plugins/cmdb_ip_picker/utils.py index 8d6b4e9f93..89ba5838bc 100644 --- a/pipeline_plugins/cmdb_ip_picker/utils.py +++ b/pipeline_plugins/cmdb_ip_picker/utils.py @@ -328,7 +328,11 @@ def fetch_host_ip_with_property_filter(self): fields.append("bk_host_innerip_v6") host_info = cmdb.get_business_host_topo( - self.username, self.bk_biz_id, self.bk_supplier_account, fields, property_filters=self.property_filters, + self.username, + self.bk_biz_id, + self.bk_supplier_account, + fields, + property_filters=self.property_filters, ) logger.info("[fetch_host_info] cmdb.get_business_host_topo return: {host_info}".format(host_info=host_info)) @@ -702,27 +706,27 @@ def get_bk_cloud_id_for_host(host_info, cloud_key="cloud"): return host_info[cloud_key][0]["id"] -def get_ges_agent_status_ipv6(bk_agent_id_list): +def get_gse_agent_status_ipv6(bk_agent_id_list): if not bk_agent_id_list: return {} ENV_MAP = {"PRODUCT": "prod", "STAGING": "stag"} gse_url = settings.BK_API_URL_TMPL.format(api_name="bk-gse") get_agent_status_url = "{}/{}/api/v2/cluster/list_agent_state".format( - gse_url, ENV_MAP.get(settings.RUN_MODE, "stag") + gse_url, ENV_MAP.get(settings.RUN_MODE, "prod") ) params = {"bk_app_code": settings.APP_CODE, "bk_app_secret": settings.SECRET_KEY, "agent_id_list": bk_agent_id_list} resp = requests.post(url=get_agent_status_url, json=params) if resp.status_code != 200: - raise Exception("[get_ges_agent_status_ipv6] 查询agent状态错误，返回值非200, content = {}".format(resp.content)) + raise Exception("[get_gse_agent_status_ipv6] 查询agent状态错误，返回值非200, content = {}".format(resp.content)) try: data = resp.json() except Exception as e: - raise Exception("[get_ges_agent_status_ipv6] 查询agent状态错误，返回值非Json, err={}".format(e)) + raise Exception("[get_gse_agent_status_ipv6] 查询agent状态错误，返回值非Json, err={}".format(e)) if data["code"] != 0: - raise Exception("[get_ges_agent_status_ipv6] 查询agent状态错误，返回值非code非0") + raise Exception("[get_gse_agent_status_ipv6] 查询agent状态错误，返回值非code非0") agent_id_status_map = {} for item in data.get("data", []): diff --git a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py index 9d129b2874..affd51d9c6 100644 --- a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py +++ b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py @@ -24,7 +24,7 @@ from gcloud.utils.handlers import handle_api_error from gcloud.utils.ip import IpRegexType, extract_ip_from_ip_str, get_ip_by_regex_type from pipeline_plugins.base.utils.inject import supplier_account_for_business, supplier_id_for_project -from pipeline_plugins.cmdb_ip_picker.utils import get_ges_agent_status_ipv6 +from pipeline_plugins.cmdb_ip_picker.utils import get_gse_agent_status_ipv6 from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6 logger = logging.getLogger("root") @@ -67,7 +67,7 @@ def match_ges_v2(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, or remote_hosts[bk_agent_id] = remote_host_value # 去查询agent状态 - agent_map = get_ges_agent_status_ipv6(bk_agent_id_list=list(remote_hosts.keys())) + agent_map = get_gse_agent_status_ipv6(bk_agent_id_list=list(remote_hosts.keys())) agent_online_ip_list = [] # 在线的ip的列表 agent_offline_ip_list = [] # 不在线的ip的列表 @@ -179,7 +179,7 @@ def get_match_ip(self): bk_agent_id_list.append(bk_agent_id) try: - agent_id_status_map = get_ges_agent_status_ipv6(bk_agent_id_list) + agent_id_status_map = get_gse_agent_status_ipv6(bk_agent_id_list) except Exception as e: raise ApiRequestError(f"ERROR:{e}")