feat(backend): 集群表头筛选以及搜索优化,实例表头筛选以及搜索优化 #3585

dbm-ui/backend/db_services/dbbase/resources/query.py

@@ -9,7 +9,9 @@
 specific language governing permissions and limitations under the License.
 """
 import abc
+import operator
 from collections import defaultdict
+from functools import reduce
 from typing import Any, Callable, Dict, List, Tuple, Union
 
 import attr
@@ -319,36 +321,79 @@ def _list_clusters(
         filter_params_map = filter_params_map or {}
         inner_filter_params_map = {
             "id": Q(id=query_params.get("id")),
-            "name": (Q(name__icontains=query_params.get("name")) | Q(alias__icontains=query_params.get("name"))),
-            "domain": Q(immute_domain__icontains=query_params.get("domain")),
-            "version": Q(major_version=query_params.get("version")),
-            "region": Q(region=query_params.get("region")),
+            "name": (
+                Q(name__in=query_params.get("name", "").split(","))
+                | Q(alias__in=query_params.get("name", "").split(","))
+            ),
+            # 版本
+            "major_version": Q(major_version__in=query_params.get("major_version", "").split(",")),
+            # 地域
+            "region": Q(region__in=query_params.get("region", "").split(",")),
             "cluster_ids": Q(id__in=query_params.get("cluster_ids")),
             "creator": Q(creator__icontains=query_params.get("creator")),
+            # 所属DB模块
+            "db_module_id": Q(db_module_id__in=query_params.get("db_module_id", "").split(",")),
+            # 管控区域
+            "bk_cloud_id": Q(bk_cloud_id__in=query_params.get("bk_cloud_id", "").split(",")),
+            # 状态
+            "status": Q(status__in=query_params.get("status", "").split(",")),
+            # 时区
+            "time_zone": Q(time_zone=query_params.get("time_zone", "").split(",")),
+            # 域名精确查询，主要用于工具箱手动填入域名查询
+            "exact_domain": Q(immute_domain=query_params.get("exact_domain")),
+            # 域名
+            "domain": Q(
+                clusterentry__cluster_entry_type=ClusterEntryType.DNS.value,
+                clusterentry__entry__in=query_params.get("domain", "").split(","),
+            ),
         }
         filter_params_map.update(inner_filter_params_map)
 
         # 通过基础过滤参数进行cluster过滤
         for param in filter_params_map:
             if query_params.get(param):
                 query_filters &= filter_params_map[param]
-        cluster_queryset = Cluster.objects.filter(query_filters)
+        cluster_queryset = Cluster.objects.filter(query_filters).order_by("create_at")
 
-        # 定义内置的过滤函数map，默认过滤函数接收这四个参数：
-        # query_params, cluster_queryset, proxy_queryset, storage_queryset
-        def filter_ip_func(_query_params, _cluster_queryset, _proxy_queryset, _storage_queryset):
-            filter_ip = query_params.get("ip").split(",")
-            _proxy_filter_ip_queryset = _proxy_queryset.filter(machine__ip__in=filter_ip)
-            _storage_filter_ip_queryset = _storage_queryset.filter(machine__ip__in=filter_ip)
+        #  部署时间表头排序
+        if query_params.get("ordering"):
+            cluster_queryset = cluster_queryset.order_by(query_params.get("ordering"))
+
+        def filter_inst_queryset(_cluster_queryset, _proxy_queryset, _storage_queryset, _filters):
+            # 注意这里用新的变量获取过滤后的queryset，不要用原queryset过滤，会影响后续集群关联实例的获取
+            _proxy_filter_queryset = _proxy_queryset.filter(_filters)
+            _storage_filter_queryset = _storage_queryset.filter(_filters)
             # 这里如果不用distinct，会查询出重复记录。TODO: 排查查询重复记录的原因
             _cluster_queryset = _cluster_queryset.filter(
-                Q(proxyinstance__in=_proxy_filter_ip_queryset) | Q(storageinstance__in=_storage_filter_ip_queryset)
+                Q(proxyinstance__in=_proxy_filter_queryset) | Q(storageinstance__in=_storage_filter_queryset)
             ).distinct()
             return _cluster_queryset
 
-        filter_func_map = filter_func_map or {}
-        filter_func_map.update(ip=filter_ip_func)
+        # ip筛选
+        def filter_ip_func(_query_params, _cluster_queryset, _proxy_queryset, _storage_queryset):
+            """实例过滤ip"""
+            filter_ip = Q(machine__ip__in=_query_params.get("ip").split(","))
+            _cluster_queryset = filter_inst_queryset(_cluster_queryset, _proxy_queryset, _storage_queryset, filter_ip)
+            return _cluster_queryset
 
+        # 实例筛选
+        def filter_instance_func(_query_params, _cluster_queryset, _proxy_queryset, _storage_queryset):
+            """实例过滤ip:port"""
+            insts = _query_params.get("instance").split(",")
+            filter_inst = reduce(
+                operator.or_, [Q(machine__ip=inst.split(":")[0], port=inst.split(":")[1]) for inst in insts]
+            )
+            _cluster_queryset = filter_inst_queryset(
+                _cluster_queryset, _proxy_queryset, _storage_queryset, filter_inst
+            )
+            return _cluster_queryset
+
+        filter_func_map = filter_func_map or {}
+        filter_func_map = {
+            "ip": filter_ip_func,
+            "instance": filter_instance_func,
+            **filter_func_map,
+        }
         # 通过基础过滤函数进行cluster过滤
         for params in filter_func_map:
             if params in query_params:
@@ -388,7 +433,7 @@ def _filter_cluster_hook(
             return ResourceList(count=0, data=[])
 
         # 预取proxy_queryset，storage_queryset，加块查询效率
-        cluster_queryset = cluster_queryset.order_by("-create_at")[offset : limit + offset].prefetch_related(
+        cluster_queryset = cluster_queryset[offset : limit + offset].prefetch_related(
             Prefetch("proxyinstance_set", queryset=proxy_queryset.select_related("machine"), to_attr="proxies"),
             Prefetch("storageinstance_set", queryset=storage_queryset.select_related("machine"), to_attr="storages"),
             "tag_set",
@@ -404,7 +449,6 @@ def _filter_cluster_hook(
             for module in DBModule.objects.filter(bk_biz_id=bk_biz_id, cluster_type__in=cls.cluster_types)
         }
 
-        # 获取集群操作记录的映射关系
         records = ClusterOperateRecord.objects.prefetch_related("ticket").filter(
             cluster_id__in=cluster_ids, ticket__status=TicketFlowStatus.RUNNING
         )
@@ -451,6 +495,7 @@ def _to_cluster_representation(
             "cluster_time_zone": cluster.time_zone,
             "cluster_name": cluster.name,
             "cluster_alias": cluster.alias,
+            "cluster_access_port": cluster.access_port,
             "cluster_type": cluster.cluster_type,
             "cluster_type_name": ClusterType.get_choice_label(cluster.cluster_type),
             "master_domain": cluster_entry.get("master_domain", ""),
@@ -492,19 +537,31 @@ def _list_instances(
         # 定义内置的过滤参数map
         inner_filter_params_map = {
             "ip": Q(machine__ip__in=query_params.get("ip", "").split(",")),
-            "port": Q(port=query_params.get("port")),
-            "status": Q(status=query_params.get("status")),
+            "port": Q(port__in=query_params.get("port", "").split(",")),
+            "status": Q(status__in=query_params.get("status", "").split(",")),
             "cluster_id": Q(cluster__id=query_params.get("cluster_id")),
             "region": Q(region=query_params.get("region")),
-            "role": Q(role=query_params.get("role")),
-            "domain": (
-                Q(cluster__immute_domain__icontains=query_params.get("domain"))
-                | Q(bind_entry__entry__icontains=query_params.get("domain"))
+            "role": Q(role__in=query_params.get("role", "").split(",")),
+            "name": Q(cluster__name__in=query_params.get("name", "").split(",")),
+            "domain": Q(
+                cluster__clusterentry__cluster_entry_type=ClusterEntryType.DNS.value,
+                cluster__clusterentry__entry__in=query_params.get("domain", "").split(","),
             ),
         }
+
+        def join_instance_by_q(instances: str) -> Q:
+            insts = instances.split(",")
+            filter_inst = reduce(
+                operator.or_, [Q(machine__ip=inst.split(":")[0], port=inst.split(":")[1]) for inst in insts]
+            )
+            return filter_inst
+
+        # 判断是否需要实例过滤
+        if query_params.get("instance"):
+            filter_params_map.update({"instance": join_instance_by_q(query_params.get("instance"))})
+
         filter_params_map = filter_params_map or {}
         filter_params_map.update(inner_filter_params_map)
-
         # 通过基础过滤参数进行instance过滤
         for param in filter_params_map:
             if query_params.get(param):
@@ -565,6 +622,7 @@ def _filter_instance_qs(cls, query_filters: Q, query_params: Dict[str, str]) ->
             "machine__bk_cloud_id",
             "machine__bk_host_id",
             "machine__spec_config",
+            "machine__machine_type",
         ]
         # 获取storage实例的查询集
         storage_queryset = (
@@ -584,7 +642,10 @@ def _filter_instance_qs(cls, query_filters: Q, query_params: Dict[str, str]) ->
 
     @classmethod
     def _filter_instance_qs_hook(cls, storage_queryset, proxy_queryset, inst_fields, query_filters, query_params):
-        instance_queryset = storage_queryset.union(proxy_queryset).values(*inst_fields).order_by("-create_at")
+        instance_queryset = storage_queryset.union(proxy_queryset).values(*inst_fields).order_by("create_at")
+        #  部署时间表头排序
+        if query_params.get("ordering"):
+            instance_queryset = instance_queryset.order_by(query_params.get("ordering"))
         return instance_queryset
 
     @classmethod
@@ -609,6 +670,7 @@ def _to_instance_representation(cls, instance: dict, cluster_entry_map: dict, **
             "port": instance["port"],
             "instance_address": f"{instance['machine__ip']}{IP_PORT_DIVIDER}{instance['port']}",
             "bk_host_id": instance["machine__bk_host_id"],
+            "machine_type": instance["machine__machine_type"],
             "role": instance["role"],
             "master_domain": cluster_entry_map.get(instance["cluster__id"], {}).get("master_domain", ""),
             "slave_domain": cluster_entry_map.get(instance["cluster__id"], {}).get("slave_domain", ""),

dbm-ui/backend/db_services/dbbase/resources/serializers.py

@@ -11,7 +11,7 @@
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
-from backend.db_meta.enums import ClusterType
+from backend.db_meta.enums import ClusterType, MachineType
 from backend.db_meta.models.cluster import Cluster
 from backend.db_services.dbbase.constants import IP_PORT_DIVIDER
 
@@ -20,13 +20,24 @@ class ListResourceSLZ(serializers.Serializer):
     id = serializers.IntegerField(required=False)
     name = serializers.CharField(required=False)
     ip = serializers.CharField(required=False)
+    instance = serializers.CharField(required=False)
     domain = serializers.CharField(required=False)
     creator = serializers.CharField(required=False)
-    version = serializers.CharField(required=False)
+    major_version = serializers.CharField(required=False)
     region = serializers.CharField(required=False)
+    cluster_ids = serializers.ListField(child=serializers.IntegerField(), required=False, allow_empty=True)
+    exact_domain = serializers.CharField(help_text=_("精确域名查询"), required=False)
+    ordering = serializers.CharField(required=False, help_text=_("排序字段,非必填"))
+    status = serializers.CharField(required=False, help_text=_("状态"))
+    db_module_id = serializers.CharField(required=False, help_text=_("所属DB模块"))
+    bk_cloud_id = serializers.CharField(required=False, help_text=_("管控区域"))
 
 
 class ListMySQLResourceSLZ(ListResourceSLZ):
+    pass
+
+
+class ListSQLServerResourceSLZ(ListResourceSLZ):
     db_module_id = serializers.IntegerField(required=False)
     cluster_ids = serializers.ListField(child=serializers.IntegerField(), required=False, allow_empty=True)
 
@@ -87,3 +98,14 @@ class RetrieveInstancesSerializer(InstanceAddressSerializer):
 class ListNodesSLZ(serializers.Serializer):
     role = serializers.CharField(help_text=_("角色"))
     keyword = serializers.CharField(help_text=_("关键字过滤"), required=False, allow_blank=True)
+
+
+class ListMachineSLZ(serializers.Serializer):
+    bk_host_id = serializers.IntegerField(help_text=_("主机ID"), required=False)
+    ip = serializers.CharField(help_text=_("IP(多个IP过滤以逗号分隔)"), required=False)
+    machine_type = serializers.ChoiceField(help_text=_("机器类型"), choices=MachineType.get_choices(), required=False)
+    bk_os_name = serializers.CharField(help_text=_("os名字"), required=False)
+    bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID"), required=False)
+    bk_agent_id = serializers.CharField(help_text=_("agent id"), required=False)
+    instance_role = serializers.CharField(help_text=_("机器部署的实例角色"), required=False)
+    creator = serializers.CharField(help_text=_("创建者"), required=False)

dbm-ui/backend/db_services/dbbase/serializers.py

@@ -9,10 +9,12 @@
 specific language governing permissions and limitations under the License.
 """
 
+from django.db.models import Q
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from backend.db_meta.enums import ClusterPhase, ClusterType
+from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource
 
 
 class IsClusterDuplicatedSerializer(serializers.Serializer):
@@ -63,3 +65,38 @@ def validate(self, attrs):
 class CommonQueryClusterResponseSerializer(serializers.Serializer):
     class Meta:
         swagger_schema_fields = {"example": []}
+
+
+class ClusterFilterSerializer(serializers.Serializer):
+    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    exact_domain = serializers.CharField(help_text=_("域名精确查询"), required=False)
+
+    # 后续有其他过滤条件可以再加
+
+    def validate(self, attrs):
+        filters = Q(bk_biz_id=attrs["bk_biz_id"])
+        filters &= Q(immute_domain=attrs["exact_domain"]) if attrs.get("exact_domain") else Q()
+        attrs["filters"] = filters
+        return attrs
+
+
+class QueryBizClusterAttrsSerializer(serializers.Serializer):
+    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    cluster_type = serializers.ChoiceField(help_text=_("集群类型"), choices=ClusterType.get_choices())
+    cluster_attrs = serializers.CharField(help_text=_("查询集群属性字段(逗号分隔)"), default="")
+    instances_attrs = serializers.CharField(help_text=_("查询实例属性字段(逗号分隔)"), default="")
+
+    def validate(self, attrs):
+        attrs["cluster_attrs"] = attrs["cluster_attrs"].split(",") if attrs["cluster_attrs"] else []
+        attrs["instances_attrs"] = attrs["instances_attrs"].split(",") if attrs["instances_attrs"] else []
+        if attrs["cluster_type"] == "redis":
+            attrs["cluster_type"] = RedisListRetrieveResource.cluster_types
+        else:
+            attrs["cluster_type"] = attrs["cluster_type"].split(",")
+
+        return attrs
+
+
+class QueryBizClusterAttrsResponseSerializer(serializers.Serializer):
+    class Meta:
+        swagger_schema_fields = {"example": {"id": [1, 2, 3], "region": ["sz", "sh"]}}