feat(mysql): tendbsingle 元数据检查 close #1241

.github/workflows/python_unit_test.yml

@@ -44,6 +44,7 @@ jobs:
         source ./dbm-ui/scripts/ci/env.sh
         export DBA_APP_BK_BIZ_ID=0
         export DB_PASSWORD=root
+        export REPORT_DB_PASSWORD=root
         export REDIS_HOST="localhost"
         export REDIS_PORT=6379
         export BROKER_URL="redis://localhost:6379/0"

dbm-ui/backend/db_periodic_task/local_tasks/ReadMe.md

@@ -0,0 +1 @@
+# 如果检查结果需要持久存储, 在 `dbm-ui/backend/db_report` 中添加 `module`

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from .db_meta_check import db_meta_check_task
+from .update_app_cache import update_app_cache
+from .update_host_dbmeta import update_host_dbmeta

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from .task import db_meta_check_task

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/check_cluster_topo.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.utils.translation import ugettext as _
+
+from backend.db_meta.enums import ClusterType, InstanceInnerRole, InstanceRole, MachineType
+from backend.db_meta.models import Cluster
+from backend.db_report.enums import MetaCheckSubType
+from backend.db_report.models import MetaCheckReport
+
+
+def check_cluster_topo():
+    _check_tendbsingle_topo()
+    _check_tendbha_topo()
+    _check_tendbcluster_topo()
+
+
+def _check_tendbsingle_topo():
+    """
+    有且只有一个存储实例
+    """
+    for c in Cluster.objects.filter(cluster_type=ClusterType.TenDBSingle):
+        messages = []
+        if c.proxyinstance_set.exists():
+            messages.append(_("有 {} 个接入层实例".format(c.proxyinstance_set.count())))
+
+        if c.storageinstance_set.count() != 1:
+            messages.append(_("有 {} 个存储层实例".format(c.storageinstance_set.count())))
+
+        ins = c.storageinstance_set.get()
+        if not (
+            ins.machine.machine_type == MachineType.SINGLE.value
+            and ins.instance_role == InstanceRole.ORPHAN.value
+            and ins.instance_inner_role == InstanceInnerRole.ORPHAN.value
+        ):
+            messages.append(
+                _(
+                    "实例 {} ({}-{}-{}) 与集群类型不匹配".format(
+                        ins.ip_port, ins.machine.machine_type, ins.instance_role, ins.instance_inner_role
+                    )
+                )
+            )
+
+        if messages:
+            MetaCheckReport.objects.create(
+                bk_biz_id=c.bk_biz_id,
+                bk_cloud_id=c.bk_cloud_id,
+                cluster=c.immute_domain,
+                cluster_type=ClusterType.TenDBSingle,
+                status=False,
+                msg=", ".join(messages),
+                subtype=MetaCheckSubType.ClusterTopo.value,
+            )
+
+
+def _check_tendbha_topo():
+    """
+    1. 至少 2 个 proxy
+    2. 1 个 master
+    3. 至少 1 个 slave
+    """
+    Cluster.objects.filter(cluster_type=ClusterType.TenDBHA)
+
+
+def _check_tendbcluster_topo():
+    Cluster.objects.filter(cluster_type=ClusterType.TenDBCluster)

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/check_instance_belong.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import Union
+
+from django.db.models import Count, Q, QuerySet
+from django.utils.translation import ugettext_lazy as _
+
+from backend.db_meta.models import ProxyInstance, StorageInstance
+from backend.db_report.enums import MetaCheckSubType
+from backend.db_report.models import MetaCheckReport
+
+
+def check_instance_belong():
+    """
+    所有实例都应该属于唯一一个集群
+    """
+    _instance_belong(StorageInstance.objects.all())
+    _instance_belong(ProxyInstance.objects.all())
+
+
+def _instance_belong(qs: QuerySet):
+    for ins in qs.annotate(cluster_count=Count("cluster")).filter(~Q(cluster_count=1)):
+        if ins.cluster.exists():  # 大于 1 个集群
+            msg = _("{} 属于 {} 个集群".format(ins.ip_port(), ins.cluster.count()))  # ToDo 详情
+        else:  # 不属于任何集群
+            msg = _("{} 不属于任何集群".format(ins.ip_port()))
+
+        MetaCheckReport.objects.create(
+            bk_biz_id=ins.bk_biz_id,
+            bk_cloud_id=ins.machine.bk_cloud_id,
+            ip=ins.machine.ip,
+            port=ins.port,
+            cluster_type=ins.cluster_type,
+            machine_type=ins.machine.machine_type,
+            status=False,
+            msg=msg,
+            subtype=MetaCheckSubType.InstanceBelong.value,
+        )

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/check_replicate_role.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.core.exceptions import ObjectDoesNotExist
+from django.utils.translation import ugettext_lazy as _
+
+from backend.db_meta.enums import InstanceInnerRole
+from backend.db_meta.models import StorageInstanceTuple
+from backend.db_report.enums import MetaCheckSubType
+from backend.db_report.models import MetaCheckReport
+
+
+def check_replicate_role():
+    """
+    ejector 只能是 master, repeater; 即不能是 slave
+    receiver 只能是 slave, repeater; 即不能是 master
+    """
+    for bad_ejector_tuple in StorageInstanceTuple.objects.filter(
+        ejector__instance_inner_role=InstanceInnerRole.SLAVE.value
+    ):
+        ejector = bad_ejector_tuple.ejector
+        try:
+            MetaCheckReport.objects.create(
+                bk_biz_id=ejector.bk_biz_id,
+                bk_cloud_id=ejector.machine.bk_cloud_id,
+                ip=ejector.machine.ip,
+                port=ejector.port,
+                cluster=ejector.cluster.get().immute_domain,
+                cluster_type=ejector.cluster_type,
+                machine_type=ejector.machine.machine_type,
+                status=False,
+                msg=_("{} {} 不能作为同步 ejector".format(ejector.ip_port(), ejector.instance_inner_role)),
+                subtype=MetaCheckSubType.ReplicateRole.value,
+            )
+        except ObjectDoesNotExist:  # 忽略实例没有集群关系的异常, instance-belong 会发现这个错误
+            pass
+
+    for bad_receiver_tuple in StorageInstanceTuple.objects.filter(
+        receiver__instance_inner_role=InstanceInnerRole.MASTER.value
+    ):
+        receiver = bad_receiver_tuple.ejector
+        try:
+            MetaCheckReport.objects.create(
+                bk_biz_id=receiver.bk_biz_id,
+                bk_cloud_id=receiver.machine.bk_cloud_id,
+                ip=receiver.machine.ip,
+                port=receiver.port,
+                cluster=receiver.cluster.get().immute_domain,
+                cluster_type=receiver.cluster_type,
+                machine_type=receiver.machine.machine_type,
+                status=False,
+                msg=_("{} {} 不能作为同步 receiver".format(receiver.ip_port(), receiver.instance_inner_role)),
+                subtype=MetaCheckSubType.ReplicateRole.value,
+            )
+        except ObjectDoesNotExist:  # 忽略实例没有集群关系的异常, instance-belong 会发现这个错误
+            pass

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/task.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from celery.schedules import crontab
+
+from backend.db_periodic_task.local_tasks.register import register_periodic_task
+
+from .check_instance_belong import check_instance_belong
+from .check_replicate_role import check_replicate_role
+
+logger = logging.getLogger("celery")
+
+
+# @register_periodic_task(run_every=crontab(minute="*/1"))
+@register_periodic_task(run_every=crontab(minute=3, hour=2))
+def db_meta_check_task():
+    """
+    巡检校验元数据
+    """
+    check_instance_belong()
+    check_replicate_role()

dbm-ui/backend/db_periodic_task/local_tasks/db_meta/update_app_cache.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import datetime
+import logging
+
+from celery.schedules import crontab
+
+from backend import env
+from backend.components import CCApi
+from backend.db_meta.models import AppCache
+from backend.db_periodic_task.local_tasks.register import register_periodic_task
+from backend.dbm_init.constants import CC_APP_ABBR_ATTR
+
+logger = logging.getLogger("celery")
+
+
+@register_periodic_task(run_every=crontab(minute="*/20"))
+def update_app_cache():
+    """缓存空闲机拓扑"""
+    now = datetime.datetime.now()
+    logger.warning("[db_meta] start update app cache start: %s", now)
+
+    bizs = CCApi.search_business().get("info", [])
+
+    updated_hosts, created_hosts = [], []
+    for biz in bizs:
+        try:
+            logger.warning("[db_meta] sync app : %s", biz["bk_biz_id"])
+            bk_app_abbr = biz.get(env.BK_APP_ABBR, "")
+            db_app_abbr = biz.get(CC_APP_ABBR_ATTR, "").lower().replace(" ", "-").replace("_", "-")
+
+            # 目标环境中存在bk_app_abbr，则同步过来
+            if env.BK_APP_ABBR and env.BK_APP_ABBR != CC_APP_ABBR_ATTR:
+                # db_app_abbr为空才同步
+                if not db_app_abbr and db_app_abbr != bk_app_abbr:
+                    CCApi.update_business(
+                        {"bk_biz_id": biz["bk_biz_id"], "data": {"db_app_abbr": bk_app_abbr}}, use_admin=True
+                    )
+                    db_app_abbr = bk_app_abbr
+
+            obj, created = AppCache.objects.update_or_create(
+                defaults={
+                    "db_app_abbr": db_app_abbr,
+                    "bk_biz_name": biz["bk_biz_name"],
+                    "language": biz["language"],
+                    "time_zone": biz["time_zone"],
+                    "bk_biz_maintainer": biz["bk_biz_maintainer"],
+                },
+                bk_biz_id=biz["bk_biz_id"],
+            )
+
+            if created:
+                created_hosts.append(obj.bk_biz_id)
+            else:
+                updated_hosts.append(obj.bk_biz_id)
+        except Exception as e:  # pylint: disable=wildcard-import
+            logger.error("[db_meta] cache app error: %s (%s)", biz, e)
+
+    logger.warning(
+        "[db_meta] finish update app cache end: %s, create_cnt: %s, update_cnt: %s",
+        datetime.datetime.now() - now,
+        len(created_hosts),
+        len(updated_hosts),
+    )