feat(backend): 集群标准化需求新增业务模块导航与mysql单节点、主从集群列表汇总接口 TencentBlueKing#6628

# Reviewed, transaction id: 22881

dbm-ui/backend/db_services/cmdb/biz.py

@@ -12,9 +12,11 @@
 import logging
 from typing import Dict, List
 
+from django.db.models import Count
+
 from backend.components import CCApi
 from backend.components.dbconfig.constants import DEPLOY_FILE_NAME, ConfType, LevelName
-from backend.db_meta.models import AppCache, DBModule
+from backend.db_meta.models import AppCache, Cluster, DBModule
 from backend.db_services.cmdb.exceptions import BkAppAttrAlreadyExistException
 from backend.db_services.dbconfig.dataclass import DBBaseConfig, DBConfigLevelData
 from backend.db_services.dbconfig.handlers import DBConfigHandler
@@ -221,3 +223,63 @@ def get_or_create_set_with_name(bk_biz_id: int, bk_set_name: str) -> int:
             raise err
         else:
             return bk_set_id
+
+
+def filter_by_biz_name(data: list, biz_name: str) -> list:
+    return [biz for biz in data if biz["bk_biz_name"] == biz_name]
+
+
+def filter_by_module_name(data: list, module_name: str) -> list:
+    result = []
+    for biz in data:
+        filtered_modules = [module for module in biz["modules"] if module["module_name"] == module_name]
+        if filtered_modules:
+            new_biz = biz.copy()
+            new_biz["modules"] = filtered_modules
+            result.append(new_biz)
+    return result
+
+
+def list_biz_module_trees(cluster_types: str, bk_biz_name: str, module_name: str) -> List[Dict]:
+    """
+    获取业务与模块维度集群数量
+    """
+
+    clusters = (
+        Cluster.objects.filter(cluster_type__in=cluster_types.split(","))
+        .values("db_module_id", "bk_biz_id")
+        .annotate(count=Count("db_module_id"))
+        .order_by("-count")
+    )
+
+    db_module_map = DBModule.db_module_map()
+    id_to_name = AppCache.id_to_name()
+
+    nested_data = collections.defaultdict(lambda: {"count": 0, "modules": collections.defaultdict(int)})
+    for cluster in clusters:
+        bk_biz_id = cluster["bk_biz_id"]
+        db_module_id = cluster["db_module_id"]
+        count = cluster["count"]
+        nested_data[bk_biz_id]["count"] += count
+        nested_data[bk_biz_id]["modules"][db_module_id] = count
+
+    final_data = []
+    for bk_biz_id, data in nested_data.items():
+        modules = [
+            {"module_name": db_module_map.get(module_id), "module_id": module_id, "count": count}
+            for module_id, count in data["modules"].items()
+        ]
+        final_data.append(
+            {
+                "bk_biz_name": id_to_name.get(bk_biz_id),
+                "bk_biz_id": bk_biz_id,
+                "count": data["count"],
+                "modules": modules,
+            }
+        )
+
+    if bk_biz_name:
+        final_data = filter_by_biz_name(final_data, bk_biz_name)
+    if module_name:
+        final_data = filter_by_module_name(final_data, module_name)
+    return final_data

dbm-ui/backend/db_services/cmdb/serializers.py

@@ -79,3 +79,21 @@ class ListNodesSerializer(TopoSerializer):
     page = serializers.IntegerField(help_text=_("页数"))
     module_id = serializers.IntegerField(help_text=_("模块ID"), required=False)
     set_id = serializers.IntegerField(help_text=_("集群ID"), required=False)
+
+
+class ListBIZModulesSLZ(serializers.Serializer):
+    cluster_types = serializers.CharField(help_text=_("集群类型(逗号分隔)"))
+    bk_biz_name = serializers.CharField(help_text=_("业务名称"), required=False)
+    module_name = serializers.CharField(help_text=_("模块名称"), required=False)
+
+
+class BIZModuleSLZ(serializers.Serializer):
+    class ModuleClusterCountSLZ(serializers.Serializer):
+        module_name = serializers.CharField(help_text=_("模块名"))
+        module_id = serializers.IntegerField(help_text=_("模块ID"))
+        count = serializers.IntegerField(help_text=_("集群数量"))
+
+    bk_biz_name = serializers.CharField(help_text=_("业务名"))
+    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    count = serializers.IntegerField(help_text=_("集群数量"))
+    modules = serializers.ListField(help_text=_("模块信息"), child=ModuleClusterCountSLZ())

dbm-ui/backend/db_services/cmdb/urls.py

@@ -16,6 +16,7 @@
 urlpatterns = [
     path("bizs/", CMDBViewSet.as_view({"get": "list_bizs"})),
     path("bizs/<int:bk_biz_id>/modules/", CMDBViewSet.as_view({"get": "list_modules"})),
+    path("biz_module_trees/", CMDBViewSet.as_view({"get": "list_biz_module_trees"})),
 ]
 
 routers = DefaultRouter(trailing_slash=True)

dbm-ui/backend/db_services/cmdb/views.py

@@ -98,3 +98,19 @@ def set_db_app_abbr(self, request, bk_biz_id):
     @action(methods=["GET"], detail=True)
     def list_cc_obj_user(self, request, bk_biz_id):
         return Response(biz.list_cc_obj_user(bk_biz_id))
+
+    @common_swagger_auto_schema(
+        operation_summary=_("业务模块树信息"),
+        query_serializer=serializers.ListBIZModulesSLZ(),
+        responses={status.HTTP_200_OK: serializers.BIZModuleSLZ(label=_("业务模块树信息"), many=True)},
+        tags=[SWAGGER_TAG],
+    )
+    @action(methods=["GET"], detail=False, serializer_class=serializers.ListBIZModulesSLZ)
+    def list_biz_module_trees(self, request):
+        cluster_types = self.params_validate(self.get_serializer_class()).get("cluster_types")
+        bk_biz_name = self.params_validate(self.get_serializer_class()).get("bk_biz_name")
+        module_name = self.params_validate(self.get_serializer_class()).get("module_name")
+        serializer = serializers.BIZModuleSLZ(
+            biz.list_biz_module_trees(cluster_types, bk_biz_name, module_name), many=True
+        )
+        return Response(serializer.data)

dbm-ui/backend/db_services/dbbase/serializers.py

@@ -17,13 +17,18 @@
 from backend.configuration.constants import DBType
 from backend.db_dirty.models import DirtyMachine
 from backend.db_meta.enums import ClusterPhase, ClusterType
+from backend.db_meta.models import DBModule
 from backend.db_services.dbbase.constants import ResourceType
+from backend.db_services.dbbase.resources.query_base import build_q_for_domain_by_cluster
 from backend.db_services.dbbase.resources.serializers import ListResourceSLZ
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
 from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource
 from backend.dbm_init.constants import CC_APP_ABBR_ATTR
 from backend.ticket.constants import TicketType
 
+db_module_id_name_map = DBModule.db_module_map()
+db_module_name_id_map = {module_name: module_id for module_id, module_name in db_module_id_name_map.items()}
+
 
 class IsClusterDuplicatedSerializer(serializers.Serializer):
     cluster_type = serializers.ChoiceField(help_text=_("集群类型"), choices=ClusterType.get_choices())
@@ -37,23 +42,57 @@ class Meta:
 
 
 class QueryAllTypeClusterSerializer(serializers.Serializer):
-    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"), required=False)
     cluster_types = serializers.CharField(help_text=_("集群类型(逗号分隔)"), required=False)
     immute_domain = serializers.CharField(help_text=_("集群域名"), required=False)
     # 额外过滤参数
     phase = serializers.ChoiceField(help_text=_("集群阶段状态"), required=False, choices=ClusterPhase.get_choices())
+    name = serializers.CharField(help_text=_("集群英文名"), required=False)
+    alias = serializers.CharField(help_text=_("集群别名"), required=False)
+    db_module_id = serializers.IntegerField(help_text=_("模块id"), required=False)
+    major_version = serializers.CharField(help_text=_("主版本号"), required=False)
+    status = serializers.CharField(help_text=_("状态"), required=False)
+    bk_cloud_id = serializers.IntegerField(help_text=_("云区域 ID"), required=False)
+    region = serializers.CharField(help_text=_("地域"), required=False)
+    db_module_name = serializers.CharField(help_text=_("模块名"), required=False)
+    cluster_type = serializers.CharField(help_text=_("集群类型"), required=False)
 
     def get_conditions(self, attr):
-        conditions = {"bk_biz_id": attr["bk_biz_id"]}
+        conditions = Q()
+
         if attr.get("cluster_types"):
-            conditions["cluster_type__in"] = attr["cluster_types"].split(",")
+            conditions &= Q(cluster_type__in=attr["cluster_types"].split(","))
+            attr.pop("cluster_types")
+
+        if attr.get("db_module_name"):
+            db_module_id = db_module_name_id_map.get(attr["db_module_name"])
+            if db_module_id is not None:
+                conditions &= Q(db_module_id=db_module_id)
+            attr.pop("db_module_name")
+
         if attr.get("immute_domain"):
-            conditions["immute_domain__icontains"] = attr["immute_domain"]
-        # 额外过滤参数
-        if attr.get("phase"):
-            conditions["phase"] = attr["phase"]
+            # 支持从域名查询
+            conditions &= build_q_for_domain_by_cluster(domains=attr.get("immute_domain", "").split(","))
+            attr.pop("immute_domain")
+
+        for field in self.fields.keys():
+            if field in attr:
+                conditions &= Q(**{field: attr[field]})
+
         return conditions
 
+    def to_representation(self, instance):
+        representation = super().to_representation(instance)
+        db_module_id = representation.get("db_module_id")
+        cloud_info = ResourceQueryHelper.search_cc_cloud(get_cache=True)
+        representation["db_module_name"] = db_module_id_name_map.get(db_module_id, "")
+        try:
+            representation["bk_cloud_name"] = cloud_info[str(representation["bk_cloud_id"])]["bk_cloud_name"]
+        except Exception:
+            representation["bk_cloud_name"] = ""
+
+        return representation
+
 
 class QueryAllTypeClusterResponseSerializer(serializers.Serializer):
     class Meta:

dbm-ui/backend/db_services/dbbase/views.py

@@ -45,7 +45,7 @@
     WebConsoleSerializer,
 )
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
-from backend.iam_app.handlers.drf_perm.base import DBManagePermission
+from backend.iam_app.handlers.drf_perm.base import DBManagePermission, ListResourcePermission
 from backend.iam_app.handlers.drf_perm.cluster import ClusterWebconsolePermission
 
 SWAGGER_TAG = _("集群通用接口")
@@ -60,11 +60,9 @@ class DBBaseViewSet(viewsets.SystemViewSet):
 
     action_permission_map = {
         ("verify_duplicated_cluster_name",): [],
-        (
-            "simple_query_cluster",
-            "common_query_cluster",
-        ): [DBManagePermission()],
+        ("common_query_cluster",): [DBManagePermission()],
         ("webconsole",): [ClusterWebconsolePermission()],
+        ("simple_query_cluster",): [ListResourcePermission()],
     }
     default_permission_class = [DBManagePermission()]
 
@@ -92,9 +90,19 @@ def verify_duplicated_cluster_name(self, request, *args, **kwargs):
     def simple_query_cluster(self, request, *args, **kwargs):
         data = self.params_validate(self.get_serializer_class())
         conditions = self.get_serializer().get_conditions(data)
-        cluster_queryset = Cluster.objects.filter(**conditions)
-        cluster_infos = [cluster.simple_desc for cluster in cluster_queryset]
-        return Response(cluster_infos)
+        cluster_queryset = Cluster.objects.filter(conditions)
+        # 兼容旧的不分页数据格式
+        if not request.query_params.get("limit"):
+            cluster_infos = [cluster.simple_desc for cluster in cluster_queryset]
+            return Response(cluster_infos)
+
+        page = self.paginate_queryset(cluster_queryset)
+        if page is not None:
+            serializer = self.get_serializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+
+        serializer = self.get_serializer(cluster_queryset, many=True)
+        return Response(serializer.data)
 
     @common_swagger_auto_schema(
         operation_summary=_("查询业务下集群通用信息"),

dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_standardize_flow.py

@@ -63,10 +63,11 @@ def standardize(self):
         增加单据临时ADMIN账号的添加和删除逻辑
         """
         cluster_ids = self.data["infos"]["cluster_ids"]
-        bk_biz_id = self.data["bk_biz_id"]
+        # 支持跨业务
+        # bk_biz_id = self.data["bk_biz_id"]
 
         cluster_objects = Cluster.objects.filter(
-            pk__in=cluster_ids, bk_biz_id=bk_biz_id, cluster_type=ClusterType.TenDBHA.value
+            pk__in=cluster_ids, cluster_type=ClusterType.TenDBHA.value
         ).prefetch_related(
             "proxyinstance_set", "storageinstance_set", "proxyinstance_set__machine", "storageinstance_set__machine"
         )

dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_standardize_flow.py

@@ -61,10 +61,11 @@ def standardize(self):
         }
         """
         cluster_ids = self.data["infos"]["cluster_ids"]
-        bk_biz_id = self.data["bk_biz_id"]
+        # 支持跨业务
+        # bk_biz_id = self.data["bk_biz_id"]
 
         cluster_objects = Cluster.objects.filter(
-            pk__in=cluster_ids, bk_biz_id=bk_biz_id, cluster_type=ClusterType.TenDBCluster.value
+            pk__in=cluster_ids, cluster_type=ClusterType.TenDBCluster.value
         ).prefetch_related(
             "proxyinstance_set",
             "storageinstance_set",

dbm-ui/backend/flow/engine/bamboo/scene/tendbsingle/standardize.py

@@ -50,10 +50,11 @@ def __init__(self, root_id: str, data: Optional[Dict]):
 
     def standardize(self):
         cluster_ids = self.data["infos"]["cluster_ids"]
-        bk_biz_id = self.data["bk_biz_id"]
+        # 支持跨业务
+        # bk_biz_id = self.data["bk_biz_id"]
 
         cluster_objects = Cluster.objects.filter(
-            pk__in=cluster_ids, bk_biz_id=bk_biz_id, cluster_type=ClusterType.TenDBSingle.value
+            pk__in=cluster_ids, cluster_type=ClusterType.TenDBSingle.value
         ).prefetch_related("storageinstance_set", "storageinstance_set__machine")
         if cluster_objects.count() != len(cluster_ids):
             raise DBMetaException(

dbm-ui/backend/iam_app/handlers/drf_perm/base.py

@@ -276,3 +276,26 @@ def has_authenticated_permission(self, request, view):
 
     def has_permission(self, request, view):
         return permissions.IsAuthenticated().has_permission(request, view)
+
+
+class ListResourcePermission(ResourceActionPermission):
+    """
+    业务相关动作的鉴权
+    """
+
+    def __init__(
+        self, actions: List[ActionMeta] = None, resource_meta: ResourceMeta = None, bk_biz_id: int = None
+    ) -> None:
+        self.bk_biz_id = bk_biz_id
+        if not self.bk_biz_id:
+            self.actions = ActionEnum.GLOBAL_MANAGE
+            self.resource_meta = None
+        else:
+            self.actions = actions or [ActionEnum.DB_MANAGE]
+            self.resource_meta = resource_meta or ResourceEnum.BUSINESS
+        super().__init__(self.actions, self.resource_meta, instance_ids_getter=self.instance_biz_id_getter)
+
+    def instance_biz_id_getter(self, request, view):
+        if not self.resource_meta:
+            return []
+        return self.get_key_id(request, view, self.resource_meta.id, many=True)

dbm-ui/backend/ticket/builders/mysql/mysql_ha_standardize.py

@@ -14,7 +14,7 @@
 from rest_framework import serializers
 
 from backend.db_meta.enums import ClusterType, InstanceInnerRole
-from backend.db_meta.models import AppCache, Cluster
+from backend.db_meta.models import Cluster
 from backend.flow.engine.controller.mysql import MySQLController
 from backend.ticket import builders
 from backend.ticket.builders.mysql.base import BaseMySQLHATicketFlowBuilder, MySQLBaseOperateDetailSerializer
@@ -25,15 +25,16 @@ class TenDBHAStandardizeDetailSerializer(MySQLBaseOperateDetailSerializer):
     class HAStandardizeDetailSerializer(serializers.Serializer):
         cluster_ids = serializers.ListField(help_text=_("集群ID列表"))
 
-    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    # 支持跨业务，bk_biz_id不需要传
+    # bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
     infos = HAStandardizeDetailSerializer(help_text=_("标准化信息"))
 
     def validate(self, attrs):
         self.__validate_clusters(attrs=attrs)
         return attrs
 
     def __validate_clusters(self, attrs):
-        AppCache.objects.get(bk_biz_id=attrs["bk_biz_id"])
+        # AppCache.objects.get(bk_biz_id=attrs["bk_biz_id"])
 
         for cluster_obj in Cluster.objects.filter(pk__in=attrs["infos"]["cluster_ids"]).all():
             if cluster_obj.cluster_type != ClusterType.TenDBHA.value:

dbm-ui/backend/ticket/builders/spider/mysql_spider_standardize.py

@@ -14,7 +14,7 @@
 from rest_framework import serializers
 
 from backend.db_meta.enums import ClusterType
-from backend.db_meta.models import AppCache, Cluster
+from backend.db_meta.models import Cluster
 from backend.flow.engine.controller.spider import SpiderController
 from backend.ticket import builders
 from backend.ticket.builders.tendbcluster.base import BaseTendbTicketFlowBuilder, TendbBaseOperateDetailSerializer
@@ -25,15 +25,16 @@ class TenDBClusterStandardizeDetailSerializer(TendbBaseOperateDetailSerializer):
     class InnerDetailSerializer(serializers.Serializer):
         cluster_ids = serializers.ListField(help_text=_("集群ID列表"))
 
-    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    # 支持跨业务,不需要传bk_biz_id
+    # bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
     infos = InnerDetailSerializer(help_text=_("标准化信息"))
 
     def validate(self, attrs):
         self.__validate_clusters(attrs=attrs)
         return attrs
 
     def __validate_clusters(self, attrs):
-        AppCache.objects.get(bk_biz_id=attrs["bk_biz_id"])
+        # AppCache.objects.get(bk_biz_id=attrs["bk_biz_id"])
 
         for cluster_obj in Cluster.objects.filter(pk__in=attrs["infos"]["cluster_ids"]).all():
             if cluster_obj.cluster_type != ClusterType.TenDBCluster.value: