feat(backend): vm单据联调 #8247

# Reviewed, transaction id: 25391

dbm-ui/backend/db_services/bigdata/urls.py

@@ -19,4 +19,5 @@
     path("bizs/<int:bk_biz_id>/pulsar/", include("backend.db_services.bigdata.pulsar.urls")),
     path("bizs/<int:bk_biz_id>/influxdb/", include("backend.db_services.bigdata.influxdb.urls")),
     path("bizs/<int:bk_biz_id>/riak/", include("backend.db_services.bigdata.riak.urls")),
+    path("bizs/<int:bk_biz_id>/vm/", include("backend.db_services.bigdata.vm.urls")),
 ]

dbm-ui/backend/db_services/bigdata/vm/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

dbm-ui/backend/db_services/bigdata/vm/constants.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+RESOURCE_TAG = "db_services/resources/vm"

dbm-ui/backend/db_services/bigdata/vm/query.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.utils.translation import ugettext_lazy as _
+
+from backend.db_meta.api.cluster.vm.detail import scan_cluster
+from backend.db_meta.enums import ClusterType, InstanceRole
+from backend.db_meta.models import Cluster, Machine
+from backend.db_services.bigdata.resources.query import BigDataBaseListRetrieveResource
+from backend.db_services.dbbase.resources.register import register_resource_decorator
+from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
+
+
+@register_resource_decorator()
+class VMListRetrieveResource(BigDataBaseListRetrieveResource):
+    cluster_types = [ClusterType.Vm]
+    instance_roles = [
+        InstanceRole.VM_STORAGE.value,
+        InstanceRole.VM_INSERT.value,
+        InstanceRole.VM_SELECT.value,
+        InstanceRole.VM_AUTH.value,
+    ]
+    fields = [
+        *BigDataBaseListRetrieveResource.fields,
+        {"name": _("Master节点"), "key": "es_master_nodes"},
+        {"name": _("热节点"), "key": "es_hot_nodes"},
+        {"name": _("冷节点"), "key": "es_cold_nodes"},
+        {"name": _("代理节点"), "key": "es_client"},
+    ]
+
+    @classmethod
+    def get_nodes(cls, bk_biz_id: int, cluster_id: int, role: str, keyword: str = None) -> list:
+        cluster = Cluster.objects.get(id=cluster_id, bk_biz_id=bk_biz_id)
+
+        storage_instances = cluster.storageinstance_set.filter(instance_role=role)
+        machines = Machine.objects.filter(bk_host_id__in=storage_instances.values_list("machine", flat=True))
+
+        role_host_ids = list(machines.values_list("bk_host_id", flat=True))
+        return ResourceQueryHelper.search_cc_hosts(role_host_ids, keyword)
+
+    @classmethod
+    def get_topo_graph(cls, bk_biz_id: int, cluster_id: int) -> dict:
+        cluster = Cluster.objects.get(bk_biz_id=bk_biz_id, id=cluster_id)
+        graph = scan_cluster(cluster).to_dict()
+        return graph

dbm-ui/backend/db_services/bigdata/vm/urls.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from rest_framework.routers import DefaultRouter
+
+from backend.db_services.bigdata.vm.views import VmClusterViewSetBigdata
+
+router = DefaultRouter(trailing_slash=True)
+router.register(r"vm_resources", VmClusterViewSetBigdata, basename="vm_resources")
+
+urlpatterns = router.urls

dbm-ui/backend/db_services/bigdata/vm/views.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.utils.decorators import method_decorator
+from django.utils.translation import ugettext_lazy as _
+from rest_framework import status
+from rest_framework.decorators import action
+from rest_framework.response import Response
+
+from backend.bk_web.swagger import common_swagger_auto_schema
+from backend.configuration.constants import DBType
+from backend.db_services.bigdata.resources import yasg_slz
+from backend.db_services.bigdata.resources.views import BigdataResourceViewSet
+from backend.db_services.bigdata.vm import constants
+from backend.db_services.bigdata.vm.query import VMListRetrieveResource
+from backend.db_services.dbbase.resources import serializers
+
+
+@method_decorator(
+    name="list",
+    decorator=common_swagger_auto_schema(
+        operation_summary=_("获取集群列表"),
+        query_serializer=serializers.ListResourceSLZ(),
+        responses={status.HTTP_200_OK: yasg_slz.PaginatedResourceSLZ()},
+        tags=[constants.RESOURCE_TAG],
+    ),
+)
+@method_decorator(
+    name="retrieve",
+    decorator=common_swagger_auto_schema(
+        operation_summary=_("获取集群详情"),
+        responses={status.HTTP_200_OK: yasg_slz.ResourceSLZ()},
+        tags=[constants.RESOURCE_TAG],
+    ),
+)
+@method_decorator(
+    name="list_instances",
+    decorator=common_swagger_auto_schema(
+        operation_summary=_("获取实例列表"),
+        query_serializer=serializers.ListInstancesSerializer(),
+        responses={status.HTTP_200_OK: yasg_slz.PaginatedResourceSLZ()},
+        tags=[constants.RESOURCE_TAG],
+    ),
+)
+@method_decorator(
+    name="retrieve_instance",
+    decorator=common_swagger_auto_schema(
+        operation_summary=_("获取实例详情"),
+        query_serializer=serializers.RetrieveInstancesSerializer(),
+        tags=[constants.RESOURCE_TAG],
+    ),
+)
+@method_decorator(
+    name="get_table_fields",
+    decorator=common_swagger_auto_schema(
+        operation_summary=_("获取查询返回字段"),
+        responses={status.HTTP_200_OK: yasg_slz.ResourceFieldSLZ()},
+        tags=[constants.RESOURCE_TAG],
+    ),
+)
+@method_decorator(
+    name="get_topo_graph",
+    decorator=common_swagger_auto_schema(
+        operation_summary=_("获取集群拓扑"),
+        responses={status.HTTP_200_OK: yasg_slz.ResourceTopoGraphSLZ()},
+        tags=[constants.RESOURCE_TAG],
+    ),
+)
+@method_decorator(
+    name="get_nodes",
+    decorator=common_swagger_auto_schema(
+        operation_summary=_("获取集群节点"),
+        query_serializer=serializers.ListNodesSLZ(),
+        tags=[constants.RESOURCE_TAG],
+    ),
+)
+class VmClusterViewSetBigdata(BigdataResourceViewSet):
+    query_class = VMListRetrieveResource
+    query_serializer_class = serializers.ListResourceSLZ
+    db_type = DBType.Vm
+
+    @action(methods=["GET"], detail=True, url_path="get_nodes", serializer_class=serializers.ListNodesSLZ)
+    def get_nodes(self, request, bk_biz_id: int, cluster_id: int):
+        """获取特定角色的节点"""
+        params = self.params_validate(self.get_serializer_class())
+        return Response(self.query_class.get_nodes(bk_biz_id, cluster_id, params["role"], params.get("keyword")))

dbm-ui/backend/db_services/dbbase/constants.py

@@ -25,6 +25,8 @@
 DORIS_DEFAULT_HTTP_PORT = 8030
 DORIS_DEFAULT_QUERY_PORT = 9030
 
+VM_INSERT_PORT = 8480
+VM_SELECT_PORT = 8481
 
 IP_PORT_DIVIDER = ":"
 SPACE_DIVIDER = " "

dbm-ui/backend/flow/consts.py

@@ -156,6 +156,9 @@
 # DORIS默认部署的实例数
 DORIS_DEFAULT_INSTANCE_NUM = 2
 
+# VictoriaMetrics默认部署实例数
+VM_DEFAULT_INSTANCE_NUM = 1
+
 # MySQL 系统内置账号列表
 MYSQL_SYS_USER = ["system user", "event_scheduler"]
 

dbm-ui/backend/iam_app/dataclass/actions.py

@@ -1026,6 +1026,29 @@ class ActionEnum:
         common_labels=[CommonActionLabel.BIZ_READ_ONLY, CommonActionLabel.BIZ_MAINTAIN, CommonActionLabel.ES_ACCESS],
     )
 
+    VM_APPLY = ActionMeta(
+        id="vm_apply",
+        name=_("VM 集群部署"),
+        name_en="vm_apply",
+        type="execute",
+        related_actions=[DB_MANAGE.id],
+        related_resource_types=[ResourceEnum.BUSINESS],
+        group=_("VM"),
+        subgroup=_("集群管理"),
+        common_labels=[CommonActionLabel.BIZ_MAINTAIN],
+    )
+
+    VM_VIEW = ActionMeta(
+        id="vm_view",
+        name=_("VictoriaMetrics 集群详情查看"),
+        name_en="vm_view",
+        type="view",
+        related_actions=[DB_MANAGE.id],
+        related_resource_types=[ResourceEnum.VM],
+        group=_("VM"),
+        subgroup=_("集群管理"),
+    )
+
     ES_ACCESS_ENTRY_VIEW = ActionMeta(
         id="es_access_entry_view",
         name=_("ES 获取访问方式"),
@@ -1050,6 +1073,17 @@ class ActionEnum:
         common_labels=[CommonActionLabel.BIZ_MAINTAIN],
     )
 
+    VM_ENABLE_DISABLE = ActionMeta(
+        id="vm_enable_disable",
+        name=_("VictoriaMetrics 集群禁用/启用"),
+        name_en="vm_enable_disable",
+        type="execute",
+        related_actions=[VM_VIEW.id],
+        related_resource_types=[ResourceEnum.VM],
+        group=_("VictoriaMetrics"),
+        subgroup=_("集群管理")
+    )
+
     DORIS_APPLY = ActionMeta(
         id=TicketType.DORIS_APPLY.lower(),
         related_resource_types=[ResourceEnum.BUSINESS],

dbm-ui/backend/ticket/builders/common/bigdata.py

@@ -278,6 +278,11 @@ class BaseDorisTicketFlowBuilder(BigDataTicketFlowBuilderPatchMixin, TicketFlowB
     cluster_types = [ClusterType.Doris.value]
 
 
+class BaseVmTicketFlowBuilder(BigDataTicketFlowBuilderPatchMixin, TicketFlowBuilder):
+    group = DBType.Vm.value
+    cluster_types = [ClusterType.Vm.value]
+
+
 class BaseHdfsTicketFlowBuilder(BigDataTicketFlowBuilderPatchMixin, TicketFlowBuilder):
     group = DBType.Hdfs.value
     cluster_types = [ClusterType.Hdfs.value]

dbm-ui/backend/ticket/builders/vm/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

dbm-ui/backend/ticket/builders/vm/vm_apply.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from django.utils.crypto import get_random_string
+from django.utils.translation import ugettext_lazy as _
+from rest_framework import serializers
+
+from backend.db_meta.enums import ClusterType
+from backend.db_services.dbbase.constants import VM_INSERT_PORT, VM_SELECT_PORT
+from backend.flow.consts import VM_DEFAULT_INSTANCE_NUM
+from backend.flow.engine.controller.vm import VmController
+from backend.ticket import builders
+from backend.ticket.builders.common.bigdata import BaseVmTicketFlowBuilder, BigDataApplyDetailsSerializer
+from backend.ticket.constants import TicketType
+
+logger = logging.getLogger("root")
+
+
+class VmApplyDetailSerializer(BigDataApplyDetailsSerializer):
+    vm_select_port = serializers.IntegerField(help_text=_("vm_select端口"), default=VM_SELECT_PORT)
+    vm_insert_port = serializers.IntegerField(help_text=_("vm_insert端口"), default=VM_INSERT_PORT)
+
+    def validate(self, attrs):
+        # 判断主机角色是否互斥
+        super().validate(attrs)
+
+        return attrs
+
+
+class VmApplyFlowParamBuilder(builders.FlowParamBuilder):
+    controller = VmController.vm_apply_scene
+
+    def format_ticket_data(self):
+        self.ticket_data.update(
+            {
+                "username": get_random_string(8),
+                "password": get_random_string(16),
+                "domain": f"vm.{self.ticket_data['cluster_name']}.{self.ticket_data['db_app_abbr']}.db",
+            }
+        )
+
+
+class VmApplyResourceParamBuilder(builders.ResourceApplyParamBuilder):
+    @classmethod
+    def fill_instance_num(cls, next_flow_data, ticket_data, nodes_key):
+        """对vm的hot和cold角色填充实例数"""
+        for role in ["hot", "cold"]:
+            if role not in next_flow_data[nodes_key]:
+                continue
+
+            for node in next_flow_data["nodes"][role]:
+                node["instance_num"] = ticket_data["resource_spec"][role].get("instance_num", VM_DEFAULT_INSTANCE_NUM)
+
+    def post_callback(self):
+        next_flow = self.ticket.next_flow()
+        self.fill_instance_num(next_flow.details["ticket_data"], self.ticket_data, nodes_key="nodes")
+        next_flow.save(update_fields=["details"])
+
+
+@builders.BuilderFactory.register(TicketType.VM_APPLY, is_apply=True, cluster_type=ClusterType.Vm)
+class VmApplyFlowBuilder(BaseVmTicketFlowBuilder):
+    serializer = VmApplyDetailSerializer
+    inner_flow_builder = VmApplyFlowParamBuilder
+    inner_flow_name = _("VictoriaMetrics 集群部署")
+    resource_apply_builder = VmApplyResourceParamBuilder

dbm-ui/backend/ticket/builders/vm/vm_destroy.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from django.utils.translation import ugettext_lazy as _
+
+from backend.db_meta.enums import ClusterPhase
+from backend.flow.engine.controller.vm import VmController
+from backend.ticket import builders
+from backend.ticket.builders.common.bigdata import BaseVmTicketFlowBuilder, BigDataTakeDownDetailSerializer
+from backend.ticket.constants import TicketType
+
+logger = logging.getLogger("root")
+
+
+class VmDestroyDetailSerializer(BigDataTakeDownDetailSerializer):
+    pass
+
+
+class VmDestroyFlowParamBuilder(builders.FlowParamBuilder):
+    controller = VmController.vm_destroy_scene
+
+
+@builders.BuilderFactory.register(TicketType.VM_DESTROY, phase=ClusterPhase.DESTROY)
+class VmDestroyFlowBuilder(BaseVmTicketFlowBuilder):
+    serializer = VmDestroyDetailSerializer
+    inner_flow_builder = VmDestroyFlowParamBuilder
+    inner_flow_name = _("VictoriaMetrics 集群下架")