feat(other): doris apply flow #3666

dbm-ui/backend/configuration/constants.py

@@ -61,6 +61,7 @@ class DBType(str, StructuredEnum):
     InfluxDB = EnumField("influxdb", _("InfluxDB"))
     Riak = EnumField("riak", _("Riak"))
     Sqlserver = EnumField("sqlserver", _("SQLServer"))
+    Doris = EnumField("doris", _("Doris"))
 
     # 不属于DB类型，仅用于云区域组件的单据部署的分组
     Cloud = EnumField("cloud", _("Cloud"))

dbm-ui/backend/db_meta/api/cluster/__init__.py

@@ -10,6 +10,7 @@
 """
 
 from . import (
+    doris,
     es,
     hdfs,
     influxdb,

dbm-ui/backend/db_meta/api/cluster/doris/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from .create import create
+from .destroy import destroy
+from .disable import disable
+from .enable import enable
+from .scale_up import scale_up
+
+# from .shrink import shrink

dbm-ui/backend/db_meta/api/cluster/doris/create.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+from typing import List, Optional
+
+from django.db import transaction
+
+from backend.db_meta import request_validator
+from backend.db_meta.api import common
+from backend.db_meta.enums import ClusterEntryType, ClusterPhase, ClusterStatus, ClusterType, InstanceRole
+from backend.db_meta.models import Cluster, ClusterEntry, StorageInstance
+from backend.flow.utils.doris.doris_module_operate import DorisCCTopoOperator
+
+logger = logging.getLogger("root")
+
+
+@transaction.atomic
+def create(
+    bk_cloud_id: int,
+    bk_biz_id: int,
+    name: str,
+    alias: str,
+    immute_domain: str,
+    db_module_id: int,
+    storages: Optional[List] = None,
+    creator: str = "",
+    major_version: str = "",
+):
+    """
+    创建 Doris 集群
+    """
+
+    bk_biz_id = request_validator.validated_integer(bk_biz_id)
+    immute_domain = request_validator.validated_domain(immute_domain)
+    db_module_id = request_validator.validated_integer(db_module_id)
+
+    # 创建集群, 添加存储和接入实例
+    cluster = Cluster.objects.create(
+        bk_biz_id=bk_biz_id,
+        name=name,
+        alias=alias,
+        cluster_type=ClusterType.Doris,
+        db_module_id=db_module_id,
+        immute_domain=immute_domain,
+        creator=creator,
+        phase=ClusterPhase.ONLINE.value,
+        status=ClusterStatus.NORMAL.value,
+        major_version=major_version,
+        bk_cloud_id=bk_cloud_id,
+    )
+
+    cluster_entry = ClusterEntry.objects.create(
+        cluster=cluster, cluster_entry_type=ClusterEntryType.DNS, entry=immute_domain, creator=creator
+    )
+
+    storages = request_validator.validated_storage_list(storages, allow_empty=False, allow_null=False)
+    storage_objs = common.filter_out_instance_obj(storages, StorageInstance.objects.all())
+    cluster.storageinstance_set.add(*storage_objs)
+
+    cluster.save()
+
+    # cluster_entry 绑定observer或follower
+    observers = common.filter_out_instance_obj(
+        storages, StorageInstance.objects.filter(instance_role=InstanceRole.DORIS_OBSERVER)
+    )
+    followers = common.filter_out_instance_obj(
+        storages, StorageInstance.objects.filter(instance_role=InstanceRole.DORIS_FOLLOWER)
+    )
+    if observers.exists():
+        cluster_entry.storageinstance_set.add(*observers)
+    else:
+        cluster_entry.storageinstance_set.add(*followers)
+    cluster_entry.save()
+
+    for ins in storage_objs:
+        m = ins.machine
+        ins.db_module_id = db_module_id
+        m.db_module_id = db_module_id
+        ins.save()
+        m.save()
+
+    # 生成域名模块、es主机转移模块、添加对应的服务实例
+    DorisCCTopoOperator(cluster).transfer_instances_to_cluster_module(storage_objs)

dbm-ui/backend/db_meta/api/cluster/doris/destroy.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from django.db import transaction
+
+from backend.configuration.constants import DBType
+from backend.db_meta.models import Cluster, ClusterEntry
+from backend.flow.utils.cc_manage import CcManage
+
+logger = logging.getLogger("root")
+
+
+@transaction.atomic
+def destroy(cluster_id: int):
+    """
+    清理DBMeta
+    """
+
+    cluster = Cluster.objects.get(id=cluster_id)
+    cc_manage = CcManage(bk_biz_id=cluster.bk_biz_id, db_type=DBType.Doris.value)
+
+    # 删除storage instance
+    for storage in cluster.storageinstance_set.all():
+        storage.delete(keep_parents=True)
+        if not storage.machine.storageinstance_set.exists():
+            # 这个 api 不需要检查返回值, 转移主机到待回收模块，转移模块这里会把服务实例删除
+            cc_manage.recycle_host([storage.machine.bk_host_id])
+            storage.machine.delete(keep_parents=True)
+
+    # 删除entry
+    for ce in ClusterEntry.objects.filter(cluster=cluster).all():
+        ce.delete(keep_parents=True)
+
+    # 删除cmdb中的模块
+    cc_manage.delete_cluster_modules(db_type=DBType.Doris.value, cluster=cluster)
+
+    # 删除集群
+    cluster.delete(keep_parents=True)

dbm-ui/backend/db_meta/api/cluster/doris/disable.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from django.db import transaction
+
+from backend.db_meta.enums import ClusterPhase
+from backend.db_meta.models import Cluster
+
+logger = logging.getLogger("root")
+
+
+@transaction.atomic
+def disable(cluster_id: int):
+    """
+    禁用Doris集群
+    """
+
+    cluster = Cluster.objects.get(id=cluster_id)
+    cluster.phase = ClusterPhase.OFFLINE.value
+    cluster.save()

dbm-ui/backend/db_meta/api/cluster/doris/enable.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from django.db import transaction
+
+from backend.db_meta.enums import ClusterPhase
+from backend.db_meta.models import Cluster
+
+logger = logging.getLogger("root")
+
+
+@transaction.atomic
+def enable(cluster_id: int):
+    """
+    启用Doris集群
+    """
+
+    cluster = Cluster.objects.get(id=cluster_id)
+    cluster.phase = ClusterPhase.ONLINE.value
+    cluster.save()

dbm-ui/backend/db_meta/api/cluster/doris/scale_up.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+from typing import List, Optional
+
+from django.db import transaction
+
+from backend.db_meta import request_validator
+from backend.db_meta.api import common
+from backend.db_meta.enums import InstanceRole
+from backend.db_meta.models import Cluster, ClusterEntry, StorageInstance
+from backend.flow.utils.doris.doris_module_operate import DorisCCTopoOperator
+
+logger = logging.getLogger("root")
+
+
+@transaction.atomic
+def scale_up(
+    cluster_id: int,
+    storages: Optional[List] = None,
+):
+    """
+    Doris集群扩容注册DBMeta
+    """
+
+    cluster = Cluster.objects.get(id=cluster_id)
+    cluster_entry = ClusterEntry.objects.get(cluster=cluster)
+
+    storages = request_validator.validated_storage_list(storages, allow_empty=False, allow_null=False)
+    storage_objs = common.filter_out_instance_obj(storages, StorageInstance.objects.all())
+    cluster.storageinstance_set.add(*storage_objs)
+
+    # 关联cluster
+    cluster.save()
+
+    # 关联对应的域名信息
+    all_observers = cluster.storageinstance_set.filter(instance_role=InstanceRole.DORIS_OBSERVER)
+    all_followers = cluster.storageinstance_set.filter(instance_role=InstanceRole.DORIS_FOLLOWER)
+
+    if all_observers.exists():
+        for instance in all_observers:
+            if not instance.bind_entry.exists():
+                cluster_entry.storageinstance_set.add(instance)
+        for instance in cluster_entry.storageinstance_set.all():
+            if instance not in all_observers:
+                instance.bind_entry.remove(cluster_entry)
+    else:
+        for instance in all_followers:
+            if not instance.bind_entry.exists():
+                cluster_entry.storageinstance_set.add(instance)
+        for instance in cluster_entry.storageinstance_set.all():
+            if instance not in all_followers:
+                instance.bind_entry.remove(cluster_entry)
+
+    cluster_entry.save()
+
+    for ins in common.filter_out_instance_obj(storages, StorageInstance.objects.all()):
+        m = ins.machine
+        ins.db_module_id = cluster.db_module_id
+        m.db_module_id = cluster.db_module_id
+        ins.save()
+        m.save()
+
+    # 生成域名模块、es主机转移模块、添加对应的服务实例
+    DorisCCTopoOperator(cluster).transfer_instances_to_cluster_module(storage_objs)

dbm-ui/backend/db_meta/enums/cluster_type.py

@@ -43,6 +43,8 @@ class ClusterType(str, StructuredEnum):
     Hdfs = EnumField("hdfs", _("Hdfs集群"))
     Influxdb = EnumField("influxdb", _("Influxdb实例"))
     Pulsar = EnumField("pulsar", _("Pulsar集群"))
+    Doris = EnumField("doris", _("Doris集群"))
+
     Dbmon = EnumField("dbmon", _("redis监控"))
 
     MongoReplicaSet = EnumField("MongoReplicaSet", _("Mongo副本集"))
@@ -85,6 +87,7 @@ def db_type_cluster_types_map(cls) -> Dict[str, List]:
             DBType.MongoDB.value: [cls.MongoShardedCluster, cls.MongoReplicaSet],
             DBType.Riak.value: [cls.Riak],
             DBType.Sqlserver.value: [cls.SqlserverHA, cls.SqlserverSingle],
+            DBType.Doris.value: [cls.Doris],
         }
 
     @classmethod

dbm-ui/backend/db_meta/enums/instance_role.py

@@ -52,6 +52,11 @@ class InstanceRole(str, StructuredEnum):
     PULSAR_ZOOKEEPER = EnumField("pulsar_zookeeper", _("pulsar_zookeeper"))
     PULSAR_BROKER = EnumField("pulsar_broker", _("pulsar_broker"))
 
+    DORIS_BACKEND_HOT = EnumField("doris_backend_hot", _("doris_backend_hot"))
+    DORIS_BACKEND_COLD = EnumField("doris_backend_cold", _("doris_backend_cold"))
+    DORIS_FOLLOWER = EnumField("doris_follower", _("doris_follower"))
+    DORIS_OBSERVER = EnumField("doris_observer", _("doris_observer"))
+
     MONGO_M1 = EnumField("mongo_m1", _("mongo_m1"))
     MONGO_M2 = EnumField("mongo_m2", _("mongo_m2"))
     MONGO_M3 = EnumField("mongo_m3", _("mongo_m3"))

dbm-ui/backend/db_meta/enums/machine_type.py

@@ -49,5 +49,9 @@ class MachineType(str, StructuredEnum):
 
     RIAK = EnumField("riak", _("riak"))
 
+    DORIS_BACKEND = EnumField("doris_backend", _("doris_backend"))
+    DORIS_FOLLOWER = EnumField("doris_follower", _("doris_follower"))
+    DORIS_OBSERVER = EnumField("doris_observer", _("doris_observer"))
+
     SQLSERVER_SINGLE = EnumField("sqlserver_single", _("sqlserver_single"))
     SQLSERVER_HA = EnumField("sqlserver_ha", _("sqlserver_ha"))

dbm-ui/backend/db_meta/enums/type_maps.py

@@ -39,6 +39,9 @@
     MachineType.RIAK: AccessLayer.STORAGE,
     MachineType.SQLSERVER_SINGLE: AccessLayer.STORAGE,
     MachineType.SQLSERVER_HA: AccessLayer.STORAGE,
+    MachineType.DORIS_FOLLOWER: AccessLayer.STORAGE,
+    MachineType.DORIS_OBSERVER: AccessLayer.STORAGE,
+    MachineType.DORIS_BACKEND: AccessLayer.STORAGE,
 }
 
 ClusterTypeMachineTypeDefine = {
@@ -65,6 +68,7 @@
     ClusterType.Riak: [MachineType.RIAK],
     ClusterType.SqlserverSingle: [MachineType.SQLSERVER_SINGLE],
     ClusterType.SqlserverHA: [MachineType.SQLSERVER_HA],
+    ClusterType.Doris: [MachineType.DORIS_BACKEND, MachineType.DORIS_FOLLOWER, MachineType.DORIS_OBSERVER],
 }
 
 ClusterMachineAccessTypeDefine = {
@@ -204,6 +208,9 @@
         InstanceRole.BACKEND_REPEATER,
         InstanceRole.BACKEND_SLAVE,
     ],
+    MachineType.DORIS_BACKEND: [InstanceRole.DORIS_BACKEND_HOT, InstanceRole.DORIS_BACKEND_COLD],
+    MachineType.DORIS_FOLLOWER: [InstanceRole.DORIS_FOLLOWER],
+    MachineType.DORIS_OBSERVER: [InstanceRole.DORIS_OBSERVER],
 }
 
 InstanceRoleInstanceInnerRoleMap = {
@@ -242,6 +249,10 @@
     InstanceRole.PULSAR_ZOOKEEPER: InstanceInnerRole.ORPHAN,
     InstanceRole.INFLUXDB: InstanceInnerRole.ORPHAN,
     InstanceRole.RIAK_NODE: InstanceInnerRole.ORPHAN,
+    InstanceRole.DORIS_FOLLOWER: InstanceInnerRole.ORPHAN,
+    InstanceRole.DORIS_OBSERVER: InstanceInnerRole.ORPHAN,
+    InstanceRole.DORIS_BACKEND_COLD: InstanceInnerRole.ORPHAN,
+    InstanceRole.DORIS_BACKEND_HOT: InstanceInnerRole.ORPHAN,
 }
 
 

dbm-ui/backend/db_meta/models/cluster_monitor.py

@@ -82,6 +82,11 @@
         MachineType.SQLSERVER_SINGLE: {"name": "sqlserver", "plugin_id": "dbm_mssql_exporter", "func_name": "mssql"},
         MachineType.SQLSERVER_HA: {"name": "sqlserver", "plugin_id": "dbm_mssql_exporter", "func_name": "mssql"},
     },
+    DBType.Doris: {
+        MachineType.DORIS_OBSERVER: {"name": "doris", "plugin_id": "dbm_doris_exporter", "func_name": "java"},
+        MachineType.DORIS_FOLLOWER: {"name": "doris", "plugin_id": "dbm_doris_exporter", "func_name": "java"},
+        MachineType.DORIS_BACKEND: {"name": "doris", "plugin_id": "dbm_doris_exporter", "func_name": "doris"},
+    },
 }
 
 SET_NAME_TEMPLATE = "db.{db_type}.{monitor_plugin_name}"