Skip to content

Commit

Permalink
feat(mysql): mysql监控探测同步关系是否符合meta同步表描述 #8481
Browse files Browse the repository at this point in the history
  • Loading branch information
xfwduke committed Dec 10, 2024
1 parent 1accc65 commit 924bcb7
Show file tree
Hide file tree
Showing 11 changed files with 299 additions and 38 deletions.
53 changes: 53 additions & 0 deletions dbm-services/mysql/db-tools/mysql-crond/pkg/nginx_updater/init.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package nginx_updater

import (
"bufio"
"os"
)

const nginxAddrFile = "/home/mysql/nginx_conf/address.list"

func Updater() {
addrs, err := readAddr()
if err != nil {

}

newAddrs, err := queryNewAddr(addrs)

f, err := os.OpenFile(nginxAddrFile, os.O_CREATE|os.O_TRUNC, 0777)
if err != nil {
}
defer func() {
_ = f.Close()
}()

for _, ad := range newAddrs {
_, _ = f.WriteString(ad + "\n")
}
}

func readAddr() (res []string, err error) {
f, err := os.Open(nginxAddrFile)
if err != nil {
return nil, err
}
defer func() {
_ = f.Close()
}()

scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
res = append(res, line)
}
err = scanner.Err()
if err != nil {
return nil, err
}
return res, nil
}

func queryNewAddr(addrs []string) (res []string, err error) {
return
}
4 changes: 2 additions & 2 deletions dbm-ui/backend/db_proxy/reverse_api/base_reverse_api_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

import logging
from types import FunctionType
from typing import Tuple
from typing import List, Tuple

from django.utils.translation import ugettext as _
from rest_framework import permissions
Expand Down Expand Up @@ -63,7 +63,7 @@ def _get_login_exempt_view_func(cls):
def get_permissions(self):
return [IPHasRegisteredPermission()]

def get_api_params(self) -> Tuple[int, str, int]:
def get_api_params(self) -> Tuple[int, str, List[int]]:
"""
return request bk_cloud_id, ip, port param
"""
Expand Down
11 changes: 11 additions & 0 deletions dbm-ui/backend/db_proxy/reverse_api/common/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from .views import CommonReverseApiView
11 changes: 11 additions & 0 deletions dbm-ui/backend/db_proxy/reverse_api/common/impl/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from .list_nginx_ips import list_nginx_ips
19 changes: 19 additions & 0 deletions dbm-ui/backend/db_proxy/reverse_api/common/impl/list_nginx_ips.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from typing import List

from backend.db_proxy.constants import ExtensionType
from backend.db_proxy.models import DBExtension


def list_nginx_ips(bk_cloud_id: int) -> List[str]:
nginx = DBExtension.get_extension_in_cloud(bk_cloud_id=bk_cloud_id, extension_type=ExtensionType.NGINX.value)
return [n.details["ip"] for n in nginx]
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,39 @@
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import logging

from django.http import JsonResponse
from django.utils.translation import ugettext_lazy as _

from backend.bk_web.swagger import common_swagger_auto_schema
from backend.db_proxy.models import DBCloudProxy
from backend.db_proxy.reverse_api.base_reverse_api_view import BaseReverseApiView
from backend.db_proxy.reverse_api.common.impl import list_nginx_ips
from backend.db_proxy.reverse_api.decorators import reverse_api

logger = logging.getLogger("root")

class CommonReverseApiView(BaseReverseApiView):
@common_swagger_auto_schema(operation_summary=_("云区域NGINX列表"))
@reverse_api(url_path="list_cloud_nginx")
def list_cloud_nginx(self, request):
bk_cloud_id, ip, port = self.get_api_params()

res = []
for ele in DBCloudProxy.objects.filter(bk_cloud_id=bk_cloud_id):
res.append(
{
"internal-address": ele.internal_address,
"external-address": ele.external_address,
}
)
class CommonReverseApiView(BaseReverseApiView):
@common_swagger_auto_schema(operation_summary=_("获取NGINX IP"))
@reverse_api(url_path="list_nginx_ips")
def list_nginx_ips(self, request, *args, **kwargs):
"""
返回特定云区域的 NGINX IP 列表
param: bk_cloud_id: int
return: ["ip1", "ip2", ...]
"""
bk_cloud_id, _, _ = self.get_api_params()
logger.info(f"bk_cloud_id: {bk_cloud_id}")
res = list_nginx_ips(bk_cloud_id=bk_cloud_id)
logger.info(f"res: {res}")

return JsonResponse({"result": True, "code": 0, "data": res, "message": "", "errors": None})
return JsonResponse(
{
"result": True,
"code": 0,
"data": res,
"message": "",
"errors": None,
}
)
12 changes: 12 additions & 0 deletions dbm-ui/backend/db_proxy/reverse_api/mysql/impl/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from .get_instance_admin_password import get_instance_admin_password
from .list_instance_info import list_instance_info
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import collections
from typing import List, Optional

from backend.components import DBPrivManagerApi
from backend.configuration.constants import DB_ADMIN_USER_MAP, DBType
from backend.db_meta.enums import ClusterType, MachineType
from backend.db_meta.models import Machine, ProxyInstance, StorageInstance
from backend.utils.string import base64_decode


def get_instance_admin_password(bk_cloud_id: int, ip: str, port_list: Optional[List[int]] = None) -> dict:
"""
目前不能正常工作
"""
m = Machine.objects.get(bk_cloud_id=bk_cloud_id, ip=ip)
if m.cluster_type in [ClusterType.TenDBSingle, ClusterType.TenDBHA]:
dbtype = DBType.MySQL
elif m.cluster_type == ClusterType.TenDBCluster:
dbtype = DBType.TenDBCluster
else:
raise Exception(f"not support cluster type: {m.cluster_type}") # noqa

if not port_list:
if m.machine_type in [MachineType.BACKEND, MachineType.REMOTE, MachineType.SINGLE]:
port_list = list(
StorageInstance.objects.filter(
machine__ip=ip,
machine__bk_cloud_id=bk_cloud_id,
).values_list("port", flat=True)
)

elif m.machine_type == MachineType.SPIDER:
port_list = list(
ProxyInstance.objects.filter(
machine__ip=ip,
machine__bk_cloud_id=bk_cloud_id,
).values_list("port", flat=True)
)
else:
raise Exception(f"not support machine type: {m.machine_type}") # noqa

instances = []
for port in port_list:
instances.append(
{
"ip": ip,
"port": port,
}
)

filters = {
"bk_biz_id": m.bk_biz_id,
"db_type": dbtype.value,
"limit": 10,
"offset": 0,
"username": DB_ADMIN_USER_MAP[dbtype],
"instances": instances,
}

admin_password_data = DBPrivManagerApi.get_mysql_admin_password(params=filters)
admin_password_data["results"] = admin_password_data.pop("items")

res = collections.defaultdict(dict)
for data in admin_password_data["results"]:
res[data["port"]]["username"] = data["username"]
res[data["port"]]["password"] = base64_decode(data["password"])

return res
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from typing import List, Optional

from django.db.models import Q

from backend.db_meta.enums import AccessLayer
from backend.db_meta.models import Machine, ProxyInstance, StorageInstance


def list_instance_info(bk_cloud_id: int, ip: str, port_list: Optional[List[int]] = None) -> List[dict]:
m = Machine.objects.get(ip=ip, bk_cloud_id=bk_cloud_id)
q = Q()
q |= Q(**{"machine": m})

if port_list:
q &= Q(**{"port__in": port_list})

if m.access_layer == AccessLayer.PROXY:
res = list_proxyinstance_info(q=q)
else:
res = list_storageinstance_info(q=q)

return res


def list_storageinstance_info(q: Q) -> List:
res = []
for i in StorageInstance.objects.filter(q):
receivers = []
ejectors = []
for t in i.as_ejector.all():
receivers.append(
{
"ip": t.receiver.machine.ip,
"port": t.receiver.port,
}
)
for t in i.as_receiver.all():
ejectors.append(
{
"ip": t.ejector.machine.ip,
"port": t.ejector.port,
}
)

res.append(
{
"ip": i.machine.ip,
"port": i.port,
"phase": i.phase,
"status": i.status,
"is_standby": i.is_stand_by,
"instance_role": i.instance_role,
"instance_inner_role": i.instance_inner_role,
"receivers": receivers,
"ejectors": ejectors,
}
)

return res


def list_proxyinstance_info(q: Q) -> List:
res = []
for i in ProxyInstance.objects.filter(q):
res.append(
{
"ip": i.machine.ip,
"port": i.port,
"phase": i.phase,
"status": i.status,
}
)

return res
21 changes: 2 additions & 19 deletions dbm-ui/backend/db_proxy/reverse_api/mysql/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,13 @@
"""
import logging

from django.db.models import Q
from django.http import JsonResponse
from django.utils.translation import ugettext_lazy as _

from backend.bk_web.swagger import common_swagger_auto_schema
from backend.db_meta.enums import AccessLayer
from backend.db_meta.models import Machine, ProxyInstance, StorageInstance
from backend.db_proxy.reverse_api.base_reverse_api_view import BaseReverseApiView
from backend.db_proxy.reverse_api.decorators import reverse_api
from backend.db_proxy.reverse_api.mysql.impl import list_instance_info

logger = logging.getLogger("root")

Expand All @@ -29,22 +27,7 @@ class MySQLReverseApiView(BaseReverseApiView):
def list_instance_info(self, request, *args, **kwargs):
bk_cloud_id, ip, port_list = self.get_api_params()
logger.info(f"bk_cloud_id: {bk_cloud_id}, ip: {ip}, port:{port_list}")

m = Machine.objects.get(ip=ip, bk_cloud_id=bk_cloud_id)
q = Q()
q |= Q(**{"machine": m})

if port_list:
q &= Q(**{"port__in": port_list})

res = []
if m.access_layer == AccessLayer.PROXY:
for i in ProxyInstance.objects.filter(q):
res.append({"ip": i.machine.ip, "port": i.port, "phase": i.phase, "status": i.status})
else:
for i in StorageInstance.objects.filter(q):
res.append({"ip": i.machine.ip, "port": i.port, "phase": i.phase, "status": i.status})

res = list_instance_info(bk_cloud_id=bk_cloud_id, ip=ip, port_list=port_list)
logger.info(f"instance info: {res}")
return JsonResponse(
{
Expand Down
Loading

0 comments on commit 924bcb7

Please sign in to comment.