feat(mysql): mysql监控探测同步关系是否符合meta同步表描述 #8481

dbm-services/mysql/db-tools/mysql-crond/pkg/nginx_updater/init.go

@@ -0,0 +1,53 @@
+package nginx_updater
+
+import (
+	"bufio"
+	"os"
+)
+
+const nginxAddrFile = "/home/mysql/nginx_conf/address.list"
+
+func Updater() {
+	addrs, err := readAddr()
+	if err != nil {
+
+	}
+
+	newAddrs, err := queryNewAddr(addrs)
+
+	f, err := os.OpenFile(nginxAddrFile, os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+
+	for _, ad := range newAddrs {
+		_, _ = f.WriteString(ad + "\n")
+	}
+}
+
+func readAddr() (res []string, err error) {
+	f, err := os.Open(nginxAddrFile)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		res = append(res, line)
+	}
+	err = scanner.Err()
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func queryNewAddr(addrs []string) (res []string, err error) {
+	return
+}

dbm-ui/backend/db_proxy/reverse_api/base_reverse_api_view.py

@@ -11,7 +11,7 @@
 
 import logging
 from types import FunctionType
-from typing import Tuple
+from typing import List, Tuple
 
 from django.utils.translation import ugettext as _
 from rest_framework import permissions
@@ -63,7 +63,7 @@ def _get_login_exempt_view_func(cls):
     def get_permissions(self):
         return [IPHasRegisteredPermission()]
 
-    def get_api_params(self) -> Tuple[int, str, int]:
+    def get_api_params(self) -> Tuple[int, str, List[int]]:
         """
         return request bk_cloud_id, ip, port param
         """

dbm-ui/backend/db_proxy/reverse_api/common/impl/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from .list_nginx_ips import list_nginx_ips

dbm-ui/backend/db_proxy/reverse_api/common/impl/list_nginx_ips.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import List
+
+from backend.db_proxy.constants import ExtensionType
+from backend.db_proxy.models import DBExtension
+
+
+def list_nginx_ips(bk_cloud_id: int) -> List[str]:
+    nginx = DBExtension.get_extension_in_cloud(bk_cloud_id=bk_cloud_id, extension_type=ExtensionType.NGINX.value)
+    return [n.details["ip"] for n in nginx]

dbm-ui/backend/db_proxy/reverse_api/common/views.py

@@ -9,15 +9,13 @@
 specific language governing permissions and limitations under the License.
 """
 import logging
-from typing import List
 
 from django.http import JsonResponse
 from django.utils.translation import ugettext_lazy as _
 
 from backend.bk_web.swagger import common_swagger_auto_schema
-from backend.db_proxy.constants import ExtensionType
-from backend.db_proxy.models import DBExtension
 from backend.db_proxy.reverse_api.base_reverse_api_view import BaseReverseApiView
+from backend.db_proxy.reverse_api.common.impl import list_nginx_ips
 from backend.db_proxy.reverse_api.decorators import reverse_api
 
 logger = logging.getLogger("root")
@@ -34,22 +32,15 @@ def list_nginx_ips(self, request, *args, **kwargs):
         """
         bk_cloud_id, _, _ = self.get_api_params()
         logger.info(f"bk_cloud_id: {bk_cloud_id}")
+        res = list_nginx_ips(bk_cloud_id=bk_cloud_id)
+        logger.info(f"res: {res}")
 
         return JsonResponse(
             {
                 "result": True,
                 "code": 0,
-                "data": self._list_nginx_ips(bk_cloud_id=bk_cloud_id),
+                "data": res,
                 "message": "",
                 "errors": None,
             }
         )
-
-    @staticmethod
-    def _list_nginx_ips(bk_cloud_id: int) -> List[str]:
-        nginx = DBExtension.get_extension_in_cloud(
-            bk_cloud_id=bk_cloud_id,
-            extension_type=ExtensionType.NGINX.value
-        )
-        logger.info(f"nginx: {nginx}")
-        return [n.details["ip"] for n in nginx]

dbm-ui/backend/db_proxy/reverse_api/mysql/impl/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from .get_instance_admin_password import get_instance_admin_password
+from .list_instance_info import list_instance_info

dbm-ui/backend/db_proxy/reverse_api/mysql/impl/get_instance_admin_password.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import collections
+from typing import List, Optional
+
+from backend.components import DBPrivManagerApi
+from backend.configuration.constants import DB_ADMIN_USER_MAP, DBType
+from backend.db_meta.enums import ClusterType, MachineType
+from backend.db_meta.models import Machine, ProxyInstance, StorageInstance
+from backend.utils.string import base64_decode
+
+
+def get_instance_admin_password(bk_cloud_id: int, ip: str, port_list: Optional[List[int]] = None) -> dict:
+    """
+    目前不能正常工作
+    """
+    m = Machine.objects.get(bk_cloud_id=bk_cloud_id, ip=ip)
+    if m.cluster_type in [ClusterType.TenDBSingle, ClusterType.TenDBHA]:
+        dbtype = DBType.MySQL
+    elif m.cluster_type == ClusterType.TenDBCluster:
+        dbtype = DBType.TenDBCluster
+    else:
+        raise Exception(f"not support cluster type: {m.cluster_type}")  # noqa
+
+    if not port_list:
+        if m.machine_type in [MachineType.BACKEND, MachineType.REMOTE, MachineType.SINGLE]:
+            port_list = list(
+                StorageInstance.objects.filter(
+                    machine__ip=ip,
+                    machine__bk_cloud_id=bk_cloud_id,
+                ).values_list("port", flat=True)
+            )
+
+        elif m.machine_type == MachineType.SPIDER:
+            port_list = list(
+                ProxyInstance.objects.filter(
+                    machine__ip=ip,
+                    machine__bk_cloud_id=bk_cloud_id,
+                ).values_list("port", flat=True)
+            )
+        else:
+            raise Exception(f"not support machine type: {m.machine_type}")  # noqa
+
+    instances = []
+    for port in port_list:
+        instances.append(
+            {
+                "ip": ip,
+                "port": port,
+            }
+        )
+
+    filters = {
+        "bk_biz_id": m.bk_biz_id,
+        "db_type": dbtype.value,
+        "limit": 10,
+        "offset": 0,
+        "username": DB_ADMIN_USER_MAP[dbtype],
+        "instances": instances,
+    }
+
+    admin_password_data = DBPrivManagerApi.get_mysql_admin_password(params=filters)
+    admin_password_data["results"] = admin_password_data.pop("items")
+
+    res = collections.defaultdict(dict)
+    for data in admin_password_data["results"]:
+        res[data["port"]]["username"] = data["username"]
+        res[data["port"]]["password"] = base64_decode(data["password"])
+
+    return res

dbm-ui/backend/db_proxy/reverse_api/mysql/impl/list_instance_info.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import List, Optional
+
+from django.db.models import Q
+
+from backend.db_meta.enums import AccessLayer
+from backend.db_meta.models import Machine, ProxyInstance, StorageInstance
+
+
+def list_instance_info(bk_cloud_id: int, ip: str, port_list: Optional[List[int]] = None) -> List[dict]:
+    m = Machine.objects.get(ip=ip, bk_cloud_id=bk_cloud_id)
+    q = Q()
+    q |= Q(**{"machine": m})
+
+    if port_list:
+        q &= Q(**{"port__in": port_list})
+
+    if m.access_layer == AccessLayer.PROXY:
+        res = list_proxyinstance_info(q=q)
+    else:
+        res = list_storageinstance_info(q=q)
+
+    return res
+
+
+def list_storageinstance_info(q: Q) -> List:
+    res = []
+    for i in StorageInstance.objects.filter(q):
+        receivers = []
+        ejectors = []
+        for t in i.as_ejector.all():
+            receivers.append(
+                {
+                    "ip": t.receiver.machine.ip,
+                    "port": t.receiver.port,
+                }
+            )
+        for t in i.as_receiver.all():
+            ejectors.append(
+                {
+                    "ip": t.ejector.machine.ip,
+                    "port": t.ejector.port,
+                }
+            )
+
+        res.append(
+            {
+                "ip": i.machine.ip,
+                "port": i.port,
+                "phase": i.phase,
+                "status": i.status,
+                "is_standby": i.is_stand_by,
+                "instance_role": i.instance_role,
+                "instance_inner_role": i.instance_inner_role,
+                "receivers": receivers,
+                "ejectors": ejectors,
+            }
+        )
+
+    return res
+
+
+def list_proxyinstance_info(q: Q) -> List:
+    res = []
+    for i in ProxyInstance.objects.filter(q):
+        res.append(
+            {
+                "ip": i.machine.ip,
+                "port": i.port,
+                "phase": i.phase,
+                "status": i.status,
+            }
+        )
+
+    return res

dbm-ui/backend/db_proxy/reverse_api/mysql/views.py

@@ -9,17 +9,14 @@
 specific language governing permissions and limitations under the License.
 """
 import logging
-from typing import List, Optional
 
-from django.db.models import Q
 from django.http import JsonResponse
 from django.utils.translation import ugettext_lazy as _
 
 from backend.bk_web.swagger import common_swagger_auto_schema
-from backend.db_meta.enums import AccessLayer
-from backend.db_meta.models import Machine, ProxyInstance, StorageInstance
 from backend.db_proxy.reverse_api.base_reverse_api_view import BaseReverseApiView
 from backend.db_proxy.reverse_api.decorators import reverse_api
+from backend.db_proxy.reverse_api.mysql.impl import list_instance_info
 
 logger = logging.getLogger("root")
 
@@ -30,22 +27,7 @@ class MySQLReverseApiView(BaseReverseApiView):
     def list_instance_info(self, request, *args, **kwargs):
         bk_cloud_id, ip, port_list = self.get_api_params()
         logger.info(f"bk_cloud_id: {bk_cloud_id}, ip: {ip}, port:{port_list}")
-
-        m = Machine.objects.get(ip=ip, bk_cloud_id=bk_cloud_id)
-        q = Q()
-        q |= Q(**{"machine": m})
-
-        if port_list:
-            q &= Q(**{"port__in": port_list})
-
-        res = []
-        if m.access_layer == AccessLayer.PROXY:
-            for i in ProxyInstance.objects.filter(q):
-                res.append({"ip": i.machine.ip, "port": i.port, "phase": i.phase, "status": i.status})
-        else:
-            for i in StorageInstance.objects.filter(q):
-                res.append({"ip": i.machine.ip, "port": i.port, "phase": i.phase, "status": i.status})
-
+        res = list_instance_info(bk_cloud_id=bk_cloud_id, ip=ip, port_list=port_list)
         logger.info(f"instance info: {res}")
         return JsonResponse(
             {
@@ -56,10 +38,3 @@ def list_instance_info(self, request, *args, **kwargs):
                 "errors": None,
             }
         )
-
-    def _list_storageinstance_info(self, ip: str, port_list: Optional[List[int]] = None) -> List:
-        pass
-
-    def _list_proxyinstance_info(self, ip: str, port_list: Optional[List[int]] = None) -> List:
-
-    pass

dbm-ui/backend/db_proxy/reverse_api/urls.py

@@ -10,8 +10,8 @@
 """
 from rest_framework.routers import DefaultRouter
 
-from backend.db_proxy.reverse_api.mysql import MySQLReverseApiView
 from backend.db_proxy.reverse_api.common import CommonReverseApiView
+from backend.db_proxy.reverse_api.mysql import MySQLReverseApiView
 
 routers = DefaultRouter(trailing_slash=True)
 routers.register("mysql", MySQLReverseApiView, basename="")