From 0d072cf055bd3c30aad6080384f18455413f4b5c Mon Sep 17 00:00:00 2001 From: ycggyao Date: Thu, 28 Nov 2024 15:33:45 +0800 Subject: [PATCH] =?UTF-8?q?feat(redis):=20redis=E5=B7=A5=E5=85=B7=E7=AE=B1?= =?UTF-8?q?=E5=8A=A0=E8=BD=BDmodule=20#7806=20#=20Reviewed,=20transaction?= =?UTF-8?q?=20id:=2025576?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../db_services/redis/redis_modules/apps.py | 33 ++++++++++ .../0004_clusterredismoduleassociate.py | 27 +++++++++ .../models/redis_module_support.py | 60 +++++++++++++++++++ .../redis/resources/redis_cluster/query.py | 6 ++ .../db_services/redis/toolbox/handlers.py | 18 ++++++ .../db_services/redis/toolbox/serializers.py | 5 ++ .../db_services/redis/toolbox/views.py | 12 ++++ .../redis/atom_jobs/redis_load_module.py | 8 +++ .../flow/utils/redis/redis_act_playload.py | 21 +++++++ .../redis/redis_toolbox_load_module.py | 51 ++++++++++++++++ dbm-ui/backend/ticket/constants.py | 16 ++++- 11 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 dbm-ui/backend/db_services/redis/redis_modules/apps.py create mode 100644 dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py create mode 100644 dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py diff --git a/dbm-ui/backend/db_services/redis/redis_modules/apps.py b/dbm-ui/backend/db_services/redis/redis_modules/apps.py new file mode 100644 index 0000000000..c7a9d684df --- /dev/null +++ b/dbm-ui/backend/db_services/redis/redis_modules/apps.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging + +from django.apps import AppConfig +from django.db.models.signals import post_migrate + +logger = logging.getLogger("root") + + +def init_default_modules(sender, **kwargs): + from backend.db_services.redis.redis_modules.models import TbRedisModuleSupport + + try: + TbRedisModuleSupport.init_default_modules() + except Exception as err: # pylint: disable=broad-except: + logger.warning(f"init_default_modules occur error, {err}") + + +class InstanceConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "backend.db_services.redis.redis_modules" + + def ready(self): + post_migrate.connect(init_default_modules, sender=self) diff --git a/dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py b/dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py new file mode 100644 index 0000000000..b3c90aecbf --- /dev/null +++ b/dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-11-07 09:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("redis_modules", "0003_alter_tbredismodulesupport_so_file"), + ] + + operations = [ + migrations.CreateModel( + name="ClusterRedisModuleAssociate", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("cluster_id", models.IntegerField(default=0, verbose_name="集群ID")), + ("module_names", models.JSONField(default=list, verbose_name="module名称列表")), + ], + options={ + "verbose_name": "Cluster Redis module关联", + "verbose_name_plural": "Cluster Redis module关联", + "db_table": "tb_cluster_redis_module_associate", + "unique_together": {("cluster_id",)}, + }, + ), + ] diff --git a/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py b/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py index ad6932d7cd..654106c581 100644 --- a/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py +++ b/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py @@ -19,3 +19,63 @@ class Meta: verbose_name_plural = _("Redis module支持") db_table = "tb_redis_module_support" unique_together = (("major_version", "module_name", "so_file"),) + + @classmethod + def init_default_modules(cls, *args, **kwargs): + """初始化module 默认数据""" + default_modules = [ + {"major_version": "Redis-4", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-4", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-4.0.9", "module_name": "fo4_lock", "so_file": "redis_fo4lock.so"}, + {"major_version": "Redis-4.0.9", "module_name": "fo4_matchmaker", "so_file": "redis_fo4matchmaker.so"}, + {"major_version": "Redis-4.0.9", "module_name": "fo4_util", "so_file": "redis_fo4util.so"}, + { + "major_version": "Redis-4.0.9", + "module_name": "jlsy-b2", + "so_file": "libB2RedisModule_linux64_service30000.so", + }, + {"major_version": "Redis-4.0.9", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-4.0.9", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-5", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-5", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-6", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-6", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-6", "module_name": "redisjson", "so_file": "librejson-2.6.6.so"}, + {"major_version": "Redis-7", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-7", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-7", "module_name": "redisjson", "so_file": "librejson-2.6.11.so"}, + ] + + # 获取已存在的记录集合 + existing_set = list( + cls.objects.filter( + major_version__in=[module["major_version"] for module in default_modules], + module_name__in=[module["module_name"] for module in default_modules], + so_file__in=[module["so_file"] for module in default_modules], + ) + .values_list("major_version", "module_name", "so_file") + .distinct() + ) + + # 准备要批量创建的新记录 + modules_to_create = [ + cls(major_version=module["major_version"], module_name=module["module_name"], so_file=module["so_file"]) + for module in default_modules + if (module["major_version"], module["module_name"], module["so_file"]) not in existing_set + ] + + # 批量创建新记录 + cls.objects.bulk_create(modules_to_create) + + +class ClusterRedisModuleAssociate(models.Model): + """redis集群-module关联表""" + + cluster_id = models.IntegerField(_("集群ID"), default=0) + module_names = models.JSONField(_("module名称列表"), default=list) + + class Meta: + verbose_name = _("Cluster Redis module关联") + verbose_name_plural = _("Cluster Redis module关联") + db_table = "tb_cluster_redis_module_associate" + unique_together = (("cluster_id"),) diff --git a/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py b/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py index ea53fa100c..60f16b244a 100644 --- a/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py +++ b/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py @@ -29,6 +29,7 @@ from backend.db_services.dbbase.resources.query import ResourceList from backend.db_services.dbbase.resources.register import register_resource_decorator from backend.db_services.ipchooser.query.resource import ResourceQueryHelper +from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate from backend.db_services.redis.resources.constants import SQL_QUERY_MASTER_SLAVE_STATUS from backend.utils.basic import dictfetchall @@ -152,6 +153,10 @@ def _to_cluster_representation( forward_to__cluster_entry_type=ClusterEntryType.CLB.value, ).exists() + # 获取集群module名称 + cluster_module = ClusterRedisModuleAssociate.objects.filter(cluster_id=cluster.id).first() + module_names = cluster_module.module_names if cluster_module is not None else [] + # 集群额外信息 cluster_extra_info = { "cluster_spec": cluster_spec, @@ -162,6 +167,7 @@ def _to_cluster_representation( "redis_slave": redis_slave, "cluster_shard_num": len(redis_master), "machine_pair_cnt": machine_pair_cnt, + "module_names": module_names, } cluster_info = super()._to_cluster_representation( cluster, diff --git a/dbm-ui/backend/db_services/redis/toolbox/handlers.py b/dbm-ui/backend/db_services/redis/toolbox/handlers.py index 7433fa84f6..33f08f8b2f 100644 --- a/dbm-ui/backend/db_services/redis/toolbox/handlers.py +++ b/dbm-ui/backend/db_services/redis/toolbox/handlers.py @@ -20,6 +20,8 @@ from backend.db_services.dbbase.cluster.handlers import ClusterServiceHandler from backend.db_services.ipchooser.handlers.host_handler import HostHandler from backend.db_services.ipchooser.query.resource import ResourceQueryHelper +from backend.db_services.redis.redis_modules.models import TbRedisModuleSupport +from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate from backend.db_services.redis.resources.constants import SQL_QUERY_COUNT_INSTANCES, SQL_QUERY_INSTANCES from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource from backend.exceptions import ApiResultError @@ -167,3 +169,19 @@ def webconsole_rpc(cls, cluster_id: int, cmd: str, db_num: int = 0, raw: bool = return {"query": "", "error_msg": err.message} return {"query": rpc_results[0]["result"], "error_msg": ""} + + @classmethod + def get_cluster_module_info(cls, cluster_id: int, version: str): + """ + 获取集群module信息 + """ + # 获取版本支持的module名称列表 + support_modules = TbRedisModuleSupport.objects.filter(major_version=version).values_list( + "module_name", flat=True + ) + # 获取集群已安装的module名称列表 + cluster_module_associate = ClusterRedisModuleAssociate.objects.filter(cluster_id=cluster_id).first() + cluster_modules = getattr(cluster_module_associate, "module_names", []) + # 字典输出集群是否安装的module列表 + results = {item: (item in cluster_modules) for item in support_modules} + return {"results": results} diff --git a/dbm-ui/backend/db_services/redis/toolbox/serializers.py b/dbm-ui/backend/db_services/redis/toolbox/serializers.py index 6e438cb98c..a33ff474d6 100644 --- a/dbm-ui/backend/db_services/redis/toolbox/serializers.py +++ b/dbm-ui/backend/db_services/redis/toolbox/serializers.py @@ -236,3 +236,8 @@ class Meta: "err_msg": "", } } + + +class GetClusterModuleInfoSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + version = serializers.CharField(help_text=_("版本")) diff --git a/dbm-ui/backend/db_services/redis/toolbox/views.py b/dbm-ui/backend/db_services/redis/toolbox/views.py index c546063c6c..0ebe13ea02 100644 --- a/dbm-ui/backend/db_services/redis/toolbox/views.py +++ b/dbm-ui/backend/db_services/redis/toolbox/views.py @@ -23,6 +23,7 @@ from backend.db_services.redis.toolbox.handlers import ToolboxHandler from backend.db_services.redis.toolbox.serializers import ( GetClusterCapacityInfoSerializer, + GetClusterModuleInfoSerializer, GetClusterVersionSerializer, QueryByOneClusterSerializer, QueryClusterIpsSerializer, @@ -95,3 +96,14 @@ def get_cluster_capacity_update_info(self, request, bk_biz_id, **kwargs): update_info = get_cluster_capacity_update_required_info(**data) update_fields = ["capacity_update_type", "require_spec_id", "require_machine_group_num", "err_msg"] return Response(dict(zip(update_fields, update_info))) + + @common_swagger_auto_schema( + operation_summary=_("获取集群module信息"), + query_serializer=GetClusterModuleInfoSerializer(), + tags=[SWAGGER_TAG], + ) + @action(methods=["GET"], detail=False, serializer_class=GetClusterModuleInfoSerializer, pagination_class=None) + def get_cluster_module_info(self, request, bk_biz_id, **kwargs): + data = self.params_validate(self.get_serializer_class()) + cluster_id, version = data["cluster_id"], data["version"] + return Response(ToolboxHandler.get_cluster_module_info(cluster_id, version)) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py b/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py index 2805490415..f5f7803412 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py @@ -137,4 +137,12 @@ def ClusterLoadModulesAtomJob(root_id, ticket_data, sub_kwargs: ActKwargs, param ) logger.info("===>redis config") + act_kwargs.get_redis_payload_func = RedisActPayload.update_cluster_module.__name__ + sub_pipeline.add_act( + act_name=_("更新cluster_module元数据"), + act_component_code=RedisConfigComponent.code, + kwargs=asdict(act_kwargs), + ) + logger.info("===>redis update_cluster_module") + return sub_pipeline.build_sub_process(sub_name=_("{}-集群加载modules").format(cluster.immute_domain)) diff --git a/dbm-ui/backend/flow/utils/redis/redis_act_playload.py b/dbm-ui/backend/flow/utils/redis/redis_act_playload.py index 6461b4f0d8..fec847ca85 100644 --- a/dbm-ui/backend/flow/utils/redis/redis_act_playload.py +++ b/dbm-ui/backend/flow/utils/redis/redis_act_playload.py @@ -35,6 +35,7 @@ get_dbmon_maxmemory_config_by_cluster_ids, ) from backend.db_services.redis.redis_dts.models.tb_tendis_dts_switch_backup import TbTendisDtsSwitchBackup +from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate from backend.db_services.redis.redis_modules.util import get_cluster_redis_modules_detial, get_redis_moudles_detail from backend.db_services.redis.util import ( is_predixy_proxy_type, @@ -2498,3 +2499,23 @@ def redis_replicas_force_resync(self, **kwargs) -> dict: "slave_ports": params["slave_ports"], }, } + + def update_cluster_module(self, cluster_map: dict) -> bool: + """ + 更新nodes cluster_module记录 + """ + cluster_id = cluster_map["cluster_id"] + new_module_names = set(cluster_map.get("load_modules", [])) + + # 获取现有记录 + obj, created = ClusterRedisModuleAssociate.objects.get_or_create( + cluster_id=cluster_id, defaults={"module_names": list(new_module_names)} + ) + if not created: + # 如果记录已存在，合并现有的模块列表和新的模块列表 + current_module_names = set(obj.module_names) + if current_module_names != new_module_names: + updated_module_names = list(current_module_names | new_module_names) + obj.module_names = updated_module_names + obj.save() + return True diff --git a/dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py new file mode 100644 index 0000000000..c1c172e71b --- /dev/null +++ b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from backend.flow.engine.controller.redis import RedisController +from backend.ticket import builders +from backend.ticket.builders.common.base import DisplayInfoSerializer, SkipToRepresentationMixin +from backend.ticket.builders.redis.base import BaseRedisTicketFlowBuilder, ClusterValidateMixin +from backend.ticket.constants import LoadConfirmType, TicketType + + +class RedisLoadModuleSerializer(SkipToRepresentationMixin, serializers.Serializer): + """redis集群加载module""" + + class InfoSerializer(DisplayInfoSerializer, ClusterValidateMixin, serializers.Serializer): + + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + db_version = serializers.CharField(help_text=_("版本号")) + load_modules = serializers.ListField( + help_text=_("module类型列表"), + child=serializers.ChoiceField( + help_text=_("module类型"), choices=LoadConfirmType.get_choices(), default=LoadConfirmType.REDIS_BLOOM + ), + required=False, + ) + + bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID")) + infos = serializers.ListField(help_text=_("批量操作参数列表"), child=InfoSerializer()) + + +class RedisLoadModuleParamBuilder(builders.FlowParamBuilder): + controller = RedisController.redis_cluster_load_modules + + def format_ticket_data(self): + super().format_ticket_data() + + +@builders.BuilderFactory.register(TicketType.REDIS_CLUSTER_LOAD_MODULES, is_apply=True) +class RedisLoadModuleFlowBuilder(BaseRedisTicketFlowBuilder): + serializer = RedisLoadModuleSerializer + inner_flow_builder = RedisLoadModuleParamBuilder + inner_flow_name = _("Redis 存量集群安装module") diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index e72dad6775..90dcadc17a 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -343,7 +343,7 @@ def get_cluster_type_by_ticket(cls, ticket_type): REDIS_CLUSTER_STORAGES_CLI_CONNS_KILL = TicketEnumField("REDIS_CLUSTER_STORAGES_CLI_CONNS_KILL", _("Redis 集群存储层cli连接kill"), register_iam=False) # noqa REDIS_CLUSTER_RENAME_DOMAIN = TicketEnumField("REDIS_CLUSTER_RENAME_DOMAIN", _("Redis集群域名重命名"), _("集群维护")) REDIS_CLUSTER_MAXMEMORY_SET = TicketEnumField("REDIS_CLUSTER_MAXMEMORY_SET", _("Redis 集群设置maxmemory")) # noqa - REDIS_CLUSTER_LOAD_MODULES = TicketEnumField("REDIS_CLUSTER_LOAD_MODULES", _("Redis 集群加载modules")) # noqa + REDIS_CLUSTER_LOAD_MODULES = TicketEnumField("REDIS_CLUSTER_LOAD_MODULES", _("Redis 集群安装modules")) # noqa REDIS_TENDISPLUS_LIGHTNING_DATA = TicketEnumField("REDIS_TENDISPLUS_LIGHTNING_DATA", _("Tendisplus闪电导入数据"), _("集群维护")) # noqa REDIS_CLUSTER_INS_MIGRATE = TicketEnumField("REDIS_CLUSTER_INS_MIGRATE", _("Redis 集群指定实例迁移"), _("集群管理")) REDIS_SINGLE_INS_MIGRATE = TicketEnumField("REDIS_SINGLE_INS_MIGRATE", _("Redis 主从指定实例迁移"), _("集群管理")) @@ -568,6 +568,20 @@ class SwitchConfirmType(str, StructuredEnum): NO_CONFIRM = EnumField("no_confirm", _("无需确认")) +class LoadConfirmType(str, StructuredEnum): + """ + 加载Module类型 + """ + + REDIS_BLOOM = EnumField("redisbloom", _("redisbloom")) + REDIS_CELL = EnumField("rediscell", _("rediscell")) + FO4_LOCK = EnumField("fo4_lock", _("fo4_lock")) + FO4_MATCHMAKER = EnumField("fo4_matchmaker", _("fo4_matchmaker")) + FO4_UTIL = EnumField("fo4_util", _("fo4_util")) + JLSY_B2 = EnumField("jlsy-b2", _("jlsy-b2")) + REDIS_JSON = EnumField("redisjson", _("redisjson")) + + class SyncDisconnectSettingType(str, StructuredEnum): """ 同步断开设置