feature: 支持gse2.0单独作为开关查询

config/default.py

@@ -155,7 +155,9 @@
 
 ENABLE_IPV6 = env.ENABLE_IPV6
 # paasv3 和 开启了ipv6 才会尝试加载 BK_API_URL_TMPL 这个变量
-if env.IS_PAAS_V3 or ENABLE_IPV6:
+ENABLE_GSE_V2 = env.ENABLE_GSE_V2
+
+if env.IS_PAAS_V3 or ENABLE_IPV6 or ENABLE_GSE_V2:
     BK_API_URL_TMPL = env.BK_APIGW_URL_TMPL
 
 if env.IS_PAAS_V3:
@@ -672,6 +674,7 @@ def monitor_report_config():
 
         from bk_monitor_report import MonitorReporter  # noqa
         from bk_monitor_report.contrib.celery import MonitorReportStep  # noqa
+
         from blueapps.core.celery import celery_app  # noqa
 
         reporter = MonitorReporter(

env.py

@@ -98,7 +98,7 @@
 ENABLE_SWAGGER_UI = os.getenv("BKAPP_ENABLE_SWAGGER_UI", False)
 
 ENABLE_IPV6 = False if os.getenv("BKAPP_ENABLE_IPV6") is None else True
-
+ENABLE_GSE_V2 = int(os.getenv("BKAPP_ENABLE_GSE_V2", 0)) == 1
 
 # 流程最高嵌套层数
 TEMPLATE_MAX_RECURSIVE_NUMBER = int(os.getenv("BKAPP_TEMPLATE_MAX_RECURSIVE_NUMBER", 200))

pipeline_plugins/cmdb_ip_picker/query.py

@@ -16,24 +16,23 @@
 import ujson as json
 from django.http import JsonResponse
 from django.utils.translation import ugettext_lazy as _
-
-from api.utils.request import batch_request
 from iam.contrib.http import HTTP_AUTH_FORBIDDEN_CODE
 from iam.exceptions import RawAuthFailedException
 
+from api.utils.request import batch_request
 from gcloud.conf import settings
 from gcloud.utils import cmdb
-from gcloud.utils.ip import format_sundry_ip
 from gcloud.utils.handlers import handle_api_error
+from gcloud.utils.ip import format_sundry_ip
 
+from .constants import ERROR_CODES, NO_ERROR
 from .utils import (
     get_cmdb_topo_tree,
-    get_objects_of_topo_tree,
-    get_modules_of_bk_obj,
+    get_gse_agent_status_ipv6,
     get_modules_id,
-    get_ges_agent_status_ipv6,
+    get_modules_of_bk_obj,
+    get_objects_of_topo_tree,
 )
-from .constants import NO_ERROR, ERROR_CODES
 
 logger = logging.getLogger("root")
 get_client_by_user = settings.ESB_GET_CLIENT_BY_USER
@@ -63,8 +62,8 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id=
     @return:
     """
     default_host_fields = ["bk_host_id", "bk_host_name", "bk_cloud_id", "bk_host_innerip"]
-    if settings.ENABLE_IPV6:
-        # IPV6环境下才会尝试去拿agent主机
+    if settings.ENABLE_IPV6 or settings.ENABLE_GSE_V2:
+        # IPV6环境下或者开启了GSE 2.0 版本
         default_host_fields.append("bk_agent_id")
     fields = set(default_host_fields + json.loads(request.GET.get("fields", "[]")))
     client = get_client_by_user(request.user.username)
@@ -130,7 +129,7 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id=
             data.append(host_detail)
 
         if "agent" in fields:
-            if settings.ENABLE_IPV6:
+            if settings.ENABLE_IPV6 or settings.ENABLE_GSE_V2:
                 # 开启IPV6将会调用网关进行查询
                 bk_agent_id_list = []
                 for host in data:
@@ -144,7 +143,7 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id=
                     bk_agent_id_list.append(bk_agent_id)
 
                 try:
-                    agent_id_status_map = get_ges_agent_status_ipv6(bk_agent_id_list)
+                    agent_id_status_map = get_gse_agent_status_ipv6(bk_agent_id_list)
                 except Exception as e:
                     result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": e}
                     return JsonResponse(result)
@@ -156,6 +155,7 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id=
                         if not host["bk_host_innerip"]:
                             # 如果既没有如果bk_agent_id，又没有ipv4地址，说明这个主机石台没有安装agent的ipv6主机，忽略，不再查询agent状态, 直接重置为未知
                             host["agent"] = -1
+                            continue
                         bk_agent_id = "{}:{}".format(host["bk_cloud_id"], host["bk_host_innerip"])
                     host["agent"] = agent_id_status_map.get(bk_agent_id, -1)
             else:

pipeline_plugins/cmdb_ip_picker/utils.py

@@ -328,7 +328,11 @@ def fetch_host_ip_with_property_filter(self):
             fields.append("bk_host_innerip_v6")
 
         host_info = cmdb.get_business_host_topo(
-            self.username, self.bk_biz_id, self.bk_supplier_account, fields, property_filters=self.property_filters,
+            self.username,
+            self.bk_biz_id,
+            self.bk_supplier_account,
+            fields,
+            property_filters=self.property_filters,
         )
         logger.info("[fetch_host_info] cmdb.get_business_host_topo return: {host_info}".format(host_info=host_info))
 
@@ -702,27 +706,27 @@ def get_bk_cloud_id_for_host(host_info, cloud_key="cloud"):
     return host_info[cloud_key][0]["id"]
 
 
-def get_ges_agent_status_ipv6(bk_agent_id_list):
+def get_gse_agent_status_ipv6(bk_agent_id_list):
     if not bk_agent_id_list:
         return {}
     ENV_MAP = {"PRODUCT": "prod", "STAGING": "stag"}
 
     gse_url = settings.BK_API_URL_TMPL.format(api_name="bk-gse")
     get_agent_status_url = "{}/{}/api/v2/cluster/list_agent_state".format(
-        gse_url, ENV_MAP.get(settings.RUN_MODE, "stag")
+        gse_url, ENV_MAP.get(settings.RUN_MODE, "prod")
     )
     params = {"bk_app_code": settings.APP_CODE, "bk_app_secret": settings.SECRET_KEY, "agent_id_list": bk_agent_id_list}
 
     resp = requests.post(url=get_agent_status_url, json=params)
 
     if resp.status_code != 200:
-        raise Exception("[get_ges_agent_status_ipv6] 查询agent状态错误，返回值非200, content = {}".format(resp.content))
+        raise Exception("[get_gse_agent_status_ipv6] 查询agent状态错误，返回值非200, content = {}".format(resp.content))
     try:
         data = resp.json()
     except Exception as e:
-        raise Exception("[get_ges_agent_status_ipv6] 查询agent状态错误，返回值非Json, err={}".format(e))
+        raise Exception("[get_gse_agent_status_ipv6] 查询agent状态错误，返回值非Json, err={}".format(e))
     if data["code"] != 0:
-        raise Exception("[get_ges_agent_status_ipv6] 查询agent状态错误，返回值非code非0")
+        raise Exception("[get_gse_agent_status_ipv6] 查询agent状态错误，返回值非code非0")
 
     agent_id_status_map = {}
     for item in data.get("data", []):

pipeline_plugins/variables/collections/sites/open/cc.py

@@ -225,7 +225,9 @@ def _self_explain(cls, **kwargs) -> List[FieldExplain]:
             FieldExplain(key="${KEY._module}", type=Type.LIST, description="集群下的模块信息列表，元素类型为字典，键为模块名，值为模块下的主机列"),
             FieldExplain(key="${KEY.flat__ip_list}", type=Type.STRING, description="本次操作创建的所有集群下的主机（去重后），用 ',' 连接"),
             FieldExplain(
-                key="${KEY.flat__verbose_ip_list}", type=Type.STRING, description="返回的是本次操作创建的所有集群下的主机（未去重），用 ',' 连接",
+                key="${KEY.flat__verbose_ip_list}",
+                type=Type.STRING,
+                description="返回的是本次操作创建的所有集群下的主机（未去重），用 ',' 连接",
             ),
             FieldExplain(
                 key="${KEY.flat__verbose_ip_module_list}",
@@ -304,7 +306,13 @@ def _handle_value_with_ipv4(username, bk_biz_id, bk_supplier_account, host_field
         if not ip_list:
             return []
 
-        hosts_list = get_business_host(username, bk_biz_id, bk_supplier_account, host_fields, ip_list,)
+        hosts_list = get_business_host(
+            username,
+            bk_biz_id,
+            bk_supplier_account,
+            host_fields,
+            ip_list,
+        )
         return hosts_list
 
     @staticmethod
@@ -408,15 +416,7 @@ def get_value(self):
 
         origin_ip_list = get_plat_ip_by_regex(origin_ips)
         filter_data = {**self.value, **self.pipeline_data}
-        if not settings.ENABLE_IPV6:
-            # 进行gse agent状态过滤
-            gse_agent_status_filter = GseAgentStatusIpFilter(origin_ip_list, filter_data)
-            match_result_ip = gse_agent_status_filter.get_match_ip()
-            if not ip_cloud:
-                return ip_separator.join(["{}".format(host["ip"]) for host in match_result_ip])
-
-            return ip_separator.join(["{}:{}".format(host["bk_cloud_id"], host["ip"]) for host in match_result_ip])
-        else:
+        if settings.ENABLE_IPV6:
             gse_agent_status_ipv6_filter = GseAgentStatusIpV6Filter(origin_ips, filter_data)
             match_result_ip = gse_agent_status_ipv6_filter.get_match_ip()
             if not ip_cloud:
@@ -430,3 +430,11 @@ def get_value(self):
                 else:
                     result.append("{}:{}".format(host["bk_cloud_id"], host["ip"]))
             return ip_separator.join(result)
+        else:
+            # 进行gse agent状态过滤
+            gse_agent_status_filter = GseAgentStatusIpFilter(origin_ip_list, filter_data)
+            match_result_ip = gse_agent_status_filter.get_match_ip()
+            if not ip_cloud:
+                return ip_separator.join(["{}".format(host["ip"]) for host in match_result_ip])
+
+            return ip_separator.join(["{}:{}".format(host["bk_cloud_id"], host["ip"]) for host in match_result_ip])

pipeline_plugins/variables/collections/sites/open/ip_filter_base.py

@@ -13,16 +13,18 @@
 import logging
 from abc import ABCMeta, abstractmethod
 
+from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from gcloud.constants import GseAgentStatus
 from gcloud.conf import settings as gcloud_settings
+from gcloud.constants import GseAgentStatus
 from gcloud.core.models import Project
 from gcloud.exceptions import ApiRequestError
+from gcloud.utils import cmdb
 from gcloud.utils.handlers import handle_api_error
-from gcloud.utils.ip import extract_ip_from_ip_str, get_ip_by_regex_type, IpRegexType
-from pipeline_plugins.base.utils.inject import supplier_id_for_project, supplier_account_for_business
-from pipeline_plugins.cmdb_ip_picker.utils import get_ges_agent_status_ipv6
+from gcloud.utils.ip import IpRegexType, extract_ip_from_ip_str, get_ip_by_regex_type
+from pipeline_plugins.base.utils.inject import supplier_account_for_business, supplier_id_for_project
+from pipeline_plugins.cmdb_ip_picker.utils import get_gse_agent_status_ipv6
 from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6
 
 logger = logging.getLogger("root")
@@ -40,18 +42,57 @@ def get_match_ip(self):
 
 
 class GseAgentStatusIpFilter(IpFilterBase):
-    def get_match_ip(self):
+    def match_ges_v2(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list):
 
-        origin_ip_list = self.origin_ip_list
-        gse_agent_status = self.data.get("gse_agent_status", "")
-        username = self.data["executor"]
-        project_id = self.data["project_id"]
-        project = Project.objects.get(id=project_id)
-        bk_biz_id = project.bk_biz_id if project.from_cmdb else ""
-        bk_supplier_id = supplier_id_for_project(project_id)
-        if not origin_ip_list:
-            return []
+        fields = ["bk_host_id", "bk_cloud_id", "bk_host_innerip", "bk_agent_id"]
+        # 生成一个host列表
+        origin_hosts = {
+            "{}:{}".format(origin_host["bk_cloud_id"], origin_host["ip"]): origin_host for origin_host in origin_ip_list
+        }
+
+        ip_list = [host["ip"] for host in origin_ip_list]
+        # 先去查出来所有host的gse_agent_id
+        hosts = cmdb.get_business_host(username, bk_biz_id, bk_supplier_id, host_fields=fields, ip_list=ip_list)
+
+        # 构造一个{bk_agent_id: bk_cloud_id:ip}的字典
+        remote_hosts = {}
+
+        for host in hosts:
+            remote_host_value = "{}:{}".format(host["bk_cloud_id"], host["bk_host_innerip"])
+            if remote_host_value in origin_hosts.keys():
+                bk_agent_id = host.get("bk_agent_id")
+                if not bk_agent_id:
+                    # 没有agent_id 使用 云区域+ip 组成 agent_id
+                    bk_agent_id = remote_host_value
+                remote_hosts[bk_agent_id] = remote_host_value
+
+        # 去查询agent状态
+        agent_map = get_gse_agent_status_ipv6(bk_agent_id_list=list(remote_hosts.keys()))
+
+        agent_online_ip_list = []  # 在线的ip的列表
+        agent_offline_ip_list = []  # 不在线的ip的列表
+        match_ip = []  # 过滤失败将不返回任何i
+        for bk_agent_id, agent_code in agent_map.items():
+            origin_host_key = remote_hosts.get(bk_agent_id)
+            if not origin_host_key:
+                continue
+            origin_host = origin_hosts.get(origin_host_key)
+            if not origin_host:
+                continue
+
+            if agent_code == GseAgentStatus.ONlINE.value:
+                agent_online_ip_list.append(origin_host)
+            if agent_code == GseAgentStatus.OFFLINE.value:
+                agent_offline_ip_list.append(origin_host)
+
+        if gse_agent_status == GseAgentStatus.ONlINE.value:
+            match_ip = agent_online_ip_list
+        if gse_agent_status == GseAgentStatus.OFFLINE.value:
+            match_ip = agent_offline_ip_list
 
+        return match_ip
+
+    def match_gse_v1(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list):
         match_ip = origin_ip_list
         if gse_agent_status in [GseAgentStatus.ONlINE.value, GseAgentStatus.OFFLINE.value]:
             client = get_client_by_user(username)
@@ -81,6 +122,23 @@ def get_match_ip(self):
 
         return match_ip
 
+    def get_match_ip(self):
+
+        origin_ip_list = self.origin_ip_list
+        gse_agent_status = self.data.get("gse_agent_status", "")
+        username = self.data["executor"]
+        project_id = self.data["project_id"]
+        project = Project.objects.get(id=project_id)
+        bk_biz_id = project.bk_biz_id if project.from_cmdb else ""
+        bk_supplier_id = supplier_id_for_project(project_id)
+        if not origin_ip_list:
+            return []
+
+        if settings.ENABLE_GSE_V2:
+            return self.match_ges_v2(gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list)
+        else:
+            return self.match_gse_v1(gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list)
+
 
 class GseAgentStatusIpV6Filter:
     def __init__(self, ip_str, data):
@@ -121,7 +179,7 @@ def get_match_ip(self):
             bk_agent_id_list.append(bk_agent_id)
 
         try:
-            agent_id_status_map = get_ges_agent_status_ipv6(bk_agent_id_list)
+            agent_id_status_map = get_gse_agent_status_ipv6(bk_agent_id_list)
         except Exception as e:
             raise ApiRequestError(f"ERROR:{e}")