From f431becded5aad13f476fa8d8c2cfb7286234376 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Tue, 5 Mar 2024 15:28:55 +0800 Subject: [PATCH] =?UTF-8?q?fix(backend):=20=E5=A2=9E=E5=8A=A0=E5=AF=B9?= =?UTF-8?q?=E5=BC=80=E5=8C=BA=E6=89=A7=E8=A1=8C=E6=A0=A1=E9=AA=8C=20#3452?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/mysql_partition/client.py | 10 ++ .../db_services/mysql/open_area/handlers.py | 118 ++++++++++++++---- .../scene/mysql/mysql_open_area_flow.py | 7 +- .../backend/flow/utils/mysql/mysql_db_meta.py | 22 ++-- .../ticket/builders/mysql/mysql_openarea.py | 4 +- 5 files changed, 126 insertions(+), 35 deletions(-) diff --git a/dbm-ui/backend/components/mysql_partition/client.py b/dbm-ui/backend/components/mysql_partition/client.py index 4740e0dc03..5347663632 100644 --- a/dbm-ui/backend/components/mysql_partition/client.py +++ b/dbm-ui/backend/components/mysql_partition/client.py @@ -59,6 +59,16 @@ def __init__(self): url="partition/disable_partition", description=_("禁用分区"), ) + self.enable_partition_cluster = self.generate_data_api( + method="POST", + url="partition/disable_partition", + description=_("禁用分区"), + ) + self.disable_partition_cluster = self.generate_data_api( + method="POST", + url="partition/disable_partition", + description=_("禁用分区"), + ) self.query_log = self.generate_data_api( method="POST", url="partition/query_log", diff --git a/dbm-ui/backend/db_services/mysql/open_area/handlers.py b/dbm-ui/backend/db_services/mysql/open_area/handlers.py index 5af5770d9f..1c8897fbf2 100644 --- a/dbm-ui/backend/db_services/mysql/open_area/handlers.py +++ b/dbm-ui/backend/db_services/mysql/open_area/handlers.py @@ -8,6 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import copy import itertools from collections import defaultdict from typing import Any, Dict, List, Union @@ -17,14 +18,16 @@ from backend.components import MySQLPrivManagerApi from backend.db_meta.enums import ClusterType from backend.db_meta.models import Cluster -from backend.db_services.mysql.open_area.exceptions import TendbOpenAreaBaseException from backend.db_services.mysql.open_area.models import TendbOpenAreaConfig from backend.db_services.mysql.permission.constants import AccountType +from backend.db_services.mysql.remote_service.handlers import RemoteServiceHandler class OpenAreaHandler: """封装开区的一些处理函数""" + ALL_TABLE_FLAG = "*all*" + @classmethod def validate_only_openarea(cls, bk_biz_id, config_name, config_id: int = -1) -> bool: """校验同一业务下的开区模板名称唯一""" @@ -33,30 +36,73 @@ def validate_only_openarea(cls, bk_biz_id, config_name, config_id: int = -1) -> return not is_duplicated @classmethod - def openarea_result_preview( - cls, operator: str, config_id: int, config_data: List[Dict[str, Union[int, str, Dict]]] - ) -> Dict[str, List[Dict[str, Any]]]: - config = TendbOpenAreaConfig.objects.get(id=config_id) - clusters = Cluster.objects.filter(id__in=[info["cluster_id"] for info in config_data]) - cluster_id__cluster = {cluster.id: cluster for cluster in clusters} + def __check_db_list(cls, source_db, real_dbs): + """检查库是否合法""" + if source_db not in real_dbs: + return _("源集群不存在库{}，请检查或修改开区模板".format(source_db)) + return "" - # 获取开区执行数据 + @classmethod + def __check_table_list(cls, real_tables, source_db, check_tables): + """检查表是否合法""" + if cls.ALL_TABLE_FLAG in check_tables: + check_tables = real_tables[source_db] + + error_msg = "" + if cls.ALL_TABLE_FLAG not in check_tables and check_tables: + not_exist_tables = set(check_tables) - set(real_tables[source_db]) + if not_exist_tables: + error_msg = _("源集群库{}中不存在表{}，请检查或修改开区模板".format(source_db, not_exist_tables)) + + return check_tables, error_msg + + @classmethod + def __get_openarea_execute_objects(cls, config, config_data, cluster_id__cluster): + """获取开区执行数据结构体""" + remote_handler = RemoteServiceHandler(bk_biz_id=config.bk_biz_id) openarea_results: List[Dict[str, Any]] = [] + + # 实时查询集群的库表 + real_dbs = remote_handler.show_databases(cluster_ids=[config.source_cluster_id])[0]["databases"] + cluster_db_infos = [{"cluster_id": config.source_cluster_id, "dbs": real_dbs}] + real_tables = remote_handler.show_tables(cluster_db_infos)[0]["table_data"] + + # 获取基础执行结构体 + execute_objects_tpl = [ + { + "source_db": config_rule["source_db"], + "target_db": config_rule["target_db_pattern"], + "schema_tblist": config_rule["schema_tblist"], + "data_tblist": config_rule["data_tblist"], + "priv_data": config_rule["priv_data"], + "authorize_ips": [], + } + for config_rule in config.config_rules + ] + # 校验每个开区执行结构，如果存在不合法的库表，则填充错误信息 + for info in execute_objects_tpl: + # 校验库是否存在 + err_db_msg = cls.__check_db_list(info["source_db"], real_dbs) + # 校验schema_tblist是否合法 + info["schema_tblist"], err_schema_tb_msg = cls.__check_table_list( + real_tables, info["source_db"], info["schema_tblist"] + ) + # 校验data_tblist是否合法 + info["data_tblist"], err_data_tb_msg = cls.__check_table_list( + real_tables, info["source_db"], info["data_tblist"] + ) + err_msg_list = [err_db_msg, err_schema_tb_msg, err_data_tb_msg] + info["error_msg"] = "\n".join([msg for msg in err_msg_list if msg]) + for data in config_data: - try: - execute_objects = [ - { - "source_db": config_rule["source_db"], - "target_db": config_rule["target_db_pattern"].format(**data["vars"]), - "schema_tblist": config_rule["schema_tblist"], - "data_tblist": config_rule["data_tblist"], - "priv_data": config_rule["priv_data"], - "authorize_ips": data["authorize_ips"], - } - for config_rule in config.config_rules - ] - except KeyError: - raise TendbOpenAreaBaseException(_("范式渲染缺少变量")) + # 获取开区执行数据 + execute_objects = copy.deepcopy(execute_objects_tpl) + for info in execute_objects: + try: + info["target_db"] = info["target_db"].format(**data["vars"]) + info["authorize_ips"] = data.get("authorize_ips", []) + except KeyError: + info["error_msg"] = info["error_msg"] + "\n" + _("范式{}渲染缺少变量".format(info["target_db"])) openarea_results.append( { @@ -66,8 +112,16 @@ def openarea_result_preview( } ) - # 获取开区授权规则 + return openarea_results + + @classmethod + def __get_openarea_rules_set(cls, config, config_data, operator, cluster_id__cluster): + """获取开区授权数据""" priv_ids = list(itertools.chain(*[rule["priv_data"] for rule in config.config_rules])) + # 如果没有授权ID，则直接返回为空 + if not priv_ids: + return [] + account_type = AccountType.TENDB if config.cluster_type == ClusterType.TenDBCluster else AccountType.MYSQL authorize_rules = MySQLPrivManagerApi.list_account_rules( {"bk_biz_id": config.bk_biz_id, "ids": priv_ids, "cluster_type": account_type} @@ -92,5 +146,21 @@ def openarea_result_preview( for data in config_data for user in user__dbs_rules.keys() ] + return authorize_details - return {"config_data": openarea_results, "rules_set": authorize_details} + @classmethod + def openarea_result_preview( + cls, operator: str, config_id: int, config_data: List[Dict[str, Union[int, str, Dict]]] + ) -> Dict[str, List[Dict[str, Any]]]: + config = TendbOpenAreaConfig.objects.get(id=config_id) + clusters = Cluster.objects.filter(id__in=[info["cluster_id"] for info in config_data]) + cluster_id__cluster = {cluster.id: cluster for cluster in clusters} + # 获取开区执行数据 + openarea_results: List[Dict[str, Any]] = cls.__get_openarea_execute_objects( + config, config_data, cluster_id__cluster + ) + # 获取开区授权规则 + rules_set: List[Dict[str, Any]] = cls.__get_openarea_rules_set( + config, config_data, operator, cluster_id__cluster + ) + return {"config_data": openarea_results, "rules_set": rules_set} diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_open_area_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_open_area_flow.py index 366fc816b1..227ad0a26b 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_open_area_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_open_area_flow.py @@ -278,8 +278,11 @@ def mysql_open_area_flow(self): if data_flag: pipeline.add_sub_pipeline(sub_flow=self.open_area_data_flow()) - # 对开区的新集群进行授权 - pipeline.add_act(act_name=_("添加mysql规则授权"), act_component_code=AuthorizeRulesComponent.code, kwargs=self.data) + # 判断是否对开区的集群进行授权 + if self.data.get("rules_set"): + pipeline.add_act( + act_name=_("添加mysql规则授权"), act_component_code=AuthorizeRulesComponent.code, kwargs=self.data + ) pipeline.run_pipeline(is_drop_random_user=True) diff --git a/dbm-ui/backend/flow/utils/mysql/mysql_db_meta.py b/dbm-ui/backend/flow/utils/mysql/mysql_db_meta.py index eda908f40e..5833932410 100644 --- a/dbm-ui/backend/flow/utils/mysql/mysql_db_meta.py +++ b/dbm-ui/backend/flow/utils/mysql/mysql_db_meta.py @@ -470,10 +470,13 @@ def mysql_cluster_offline(self): cluster = Cluster.objects.get(id=self.cluster["id"]) cluster.phase = ClusterPhase.OFFLINE cluster.save() - # TODO: 修改分区配置为禁用状态 - DBPartitionApi.disable_partition( - params={"cluster_type": cluster.cluster_type, "bk_biz_id": self.bk_biz_id, "cluster_ids": [cluster.id]} - ) + # 修改分区配置为禁用状态 - offlinewithclu + disable_partition_params = { + "cluster_type": cluster.cluster_type, + "operator": self.ticket_data["creator"], + "cluster_ids": [cluster.id], + } + DBPartitionApi.disable_partition_cluster(params=disable_partition_params) def mysql_cluster_online(self): """ @@ -482,10 +485,13 @@ def mysql_cluster_online(self): cluster = Cluster.objects.get(id=self.cluster["id"]) cluster.phase = ClusterPhase.ONLINE cluster.save() - # TODO: 修改分区配置为启用状态 - DBPartitionApi.enable_partition( - params={"cluster_type": cluster.cluster_type, "bk_biz_id": self.bk_biz_id, "cluster_ids": [cluster.id]} - ) + # 修改分区配置为启用状态 - online + disable_partition_params = { + "cluster_type": cluster.cluster_type, + "operator": self.ticket_data["creator"], + "cluster_ids": [cluster.id], + } + DBPartitionApi.enable_partition_cluster(params=disable_partition_params) def mysql_migrate_cluster_switch_storage(self): """ diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_openarea.py b/dbm-ui/backend/ticket/builders/mysql/mysql_openarea.py index 17d7bfa259..3fc3b17a3c 100644 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_openarea.py +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_openarea.py @@ -45,7 +45,9 @@ class AccountRulesSerializer(serializers.Serializer): cluster_id = serializers.IntegerField(help_text=_("源集群ID")) force = serializers.BooleanField(help_text=_("是否强制执行"), required=False, default=False) config_data = serializers.ListSerializer(help_text=_("分区信息"), child=ConfigDataSerializer()) - rules_set = serializers.ListSerializer(help_text=_("授权信息"), child=PrivDataSerializer()) + rules_set = serializers.ListSerializer( + help_text=_("授权信息"), child=PrivDataSerializer(), required=False, allow_null=True, allow_empty=True + ) class MysqlOpenAreaParamBuilder(builders.FlowParamBuilder):