feat(redis): redis工具箱加载module #7806

# Reviewed, transaction id: 25576

dbm-ui/backend/db_services/redis/redis_modules/apps.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from django.apps import AppConfig
+from django.db.models.signals import post_migrate
+
+logger = logging.getLogger("root")
+
+
+def init_default_modules(sender, **kwargs):
+    from backend.db_services.redis.redis_modules.models import TbRedisModuleSupport
+
+    try:
+        TbRedisModuleSupport.init_default_modules()
+    except Exception as err:  # pylint: disable=broad-except:
+        logger.warning(f"init_default_modules occur error, {err}")
+
+
+class InstanceConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "backend.db_services.redis.redis_modules"
+
+    def ready(self):
+        post_migrate.connect(init_default_modules, sender=self)

dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.25 on 2024-11-07 09:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("redis_modules", "0003_alter_tbredismodulesupport_so_file"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ClusterRedisModuleAssociate",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("cluster_id", models.IntegerField(default=0, verbose_name="集群ID")),
+                ("module_names", models.JSONField(default=list, verbose_name="module名称列表")),
+            ],
+            options={
+                "verbose_name": "Cluster Redis module关联",
+                "verbose_name_plural": "Cluster Redis module关联",
+                "db_table": "tb_cluster_redis_module_associate",
+                "unique_together": {("cluster_id",)},
+            },
+        ),
+    ]

dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py

@@ -19,3 +19,63 @@ class Meta:
         verbose_name_plural = _("Redis module支持")
         db_table = "tb_redis_module_support"
         unique_together = (("major_version", "module_name", "so_file"),)
+
+    @classmethod
+    def init_default_modules(cls, *args, **kwargs):
+        """初始化module 默认数据"""
+        default_modules = [
+            {"major_version": "Redis-4", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"},
+            {"major_version": "Redis-4", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"},
+            {"major_version": "Redis-4.0.9", "module_name": "fo4_lock", "so_file": "redis_fo4lock.so"},
+            {"major_version": "Redis-4.0.9", "module_name": "fo4_matchmaker", "so_file": "redis_fo4matchmaker.so"},
+            {"major_version": "Redis-4.0.9", "module_name": "fo4_util", "so_file": "redis_fo4util.so"},
+            {
+                "major_version": "Redis-4.0.9",
+                "module_name": "jlsy-b2",
+                "so_file": "libB2RedisModule_linux64_service30000.so",
+            },
+            {"major_version": "Redis-4.0.9", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"},
+            {"major_version": "Redis-4.0.9", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"},
+            {"major_version": "Redis-5", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"},
+            {"major_version": "Redis-5", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"},
+            {"major_version": "Redis-6", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"},
+            {"major_version": "Redis-6", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"},
+            {"major_version": "Redis-6", "module_name": "redisjson", "so_file": "librejson-2.6.6.so"},
+            {"major_version": "Redis-7", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"},
+            {"major_version": "Redis-7", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"},
+            {"major_version": "Redis-7", "module_name": "redisjson", "so_file": "librejson-2.6.11.so"},
+        ]
+
+        # 获取已存在的记录集合
+        existing_set = list(
+            cls.objects.filter(
+                major_version__in=[module["major_version"] for module in default_modules],
+                module_name__in=[module["module_name"] for module in default_modules],
+                so_file__in=[module["so_file"] for module in default_modules],
+            )
+            .values_list("major_version", "module_name", "so_file")
+            .distinct()
+        )
+
+        # 准备要批量创建的新记录
+        modules_to_create = [
+            cls(major_version=module["major_version"], module_name=module["module_name"], so_file=module["so_file"])
+            for module in default_modules
+            if (module["major_version"], module["module_name"], module["so_file"]) not in existing_set
+        ]
+
+        # 批量创建新记录
+        cls.objects.bulk_create(modules_to_create)
+
+
+class ClusterRedisModuleAssociate(models.Model):
+    """redis集群-module关联表"""
+
+    cluster_id = models.IntegerField(_("集群ID"), default=0)
+    module_names = models.JSONField(_("module名称列表"), default=list)
+
+    class Meta:
+        verbose_name = _("Cluster Redis module关联")
+        verbose_name_plural = _("Cluster Redis module关联")
+        db_table = "tb_cluster_redis_module_associate"
+        unique_together = (("cluster_id"),)

dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py

@@ -29,6 +29,7 @@
 from backend.db_services.dbbase.resources.query import ResourceList
 from backend.db_services.dbbase.resources.register import register_resource_decorator
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
+from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate
 from backend.db_services.redis.resources.constants import SQL_QUERY_MASTER_SLAVE_STATUS
 from backend.utils.basic import dictfetchall
 
@@ -152,6 +153,10 @@ def _to_cluster_representation(
             forward_to__cluster_entry_type=ClusterEntryType.CLB.value,
         ).exists()
 
+        # 获取集群module名称
+        cluster_module = ClusterRedisModuleAssociate.objects.filter(cluster_id=cluster.id).first()
+        module_names = cluster_module.module_names if cluster_module is not None else []
+
         # 集群额外信息
         cluster_extra_info = {
             "cluster_spec": cluster_spec,
@@ -162,6 +167,7 @@ def _to_cluster_representation(
             "redis_slave": redis_slave,
             "cluster_shard_num": len(redis_master),
             "machine_pair_cnt": machine_pair_cnt,
+            "module_names": module_names,
         }
         cluster_info = super()._to_cluster_representation(
             cluster,

dbm-ui/backend/db_services/redis/toolbox/handlers.py

@@ -20,6 +20,8 @@
 from backend.db_services.dbbase.cluster.handlers import ClusterServiceHandler
 from backend.db_services.ipchooser.handlers.host_handler import HostHandler
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
+from backend.db_services.redis.redis_modules.models import TbRedisModuleSupport
+from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate
 from backend.db_services.redis.resources.constants import SQL_QUERY_COUNT_INSTANCES, SQL_QUERY_INSTANCES
 from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource
 from backend.exceptions import ApiResultError
@@ -167,3 +169,19 @@ def webconsole_rpc(cls, cluster_id: int, cmd: str, db_num: int = 0, raw: bool =
             return {"query": "", "error_msg": err.message}
 
         return {"query": rpc_results[0]["result"], "error_msg": ""}
+
+    @classmethod
+    def get_cluster_module_info(cls, cluster_id: int, version: str):
+        """
+        获取集群module信息
+        """
+        # 获取版本支持的module名称列表
+        support_modules = TbRedisModuleSupport.objects.filter(major_version=version).values_list(
+            "module_name", flat=True
+        )
+        # 获取集群已安装的module名称列表
+        cluster_module_associate = ClusterRedisModuleAssociate.objects.filter(cluster_id=cluster_id).first()
+        cluster_modules = getattr(cluster_module_associate, "module_names", [])
+        # 字典输出集群是否安装的module列表
+        results = {item: (item in cluster_modules) for item in support_modules}
+        return {"results": results}

dbm-ui/backend/db_services/redis/toolbox/serializers.py

@@ -236,3 +236,8 @@ class Meta:
                 "err_msg": "",
             }
         }
+
+
+class GetClusterModuleInfoSerializer(serializers.Serializer):
+    cluster_id = serializers.IntegerField(help_text=_("集群ID"))
+    version = serializers.CharField(help_text=_("版本"))

dbm-ui/backend/db_services/redis/toolbox/views.py

@@ -23,6 +23,7 @@
 from backend.db_services.redis.toolbox.handlers import ToolboxHandler
 from backend.db_services.redis.toolbox.serializers import (
     GetClusterCapacityInfoSerializer,
+    GetClusterModuleInfoSerializer,
     GetClusterVersionSerializer,
     QueryByOneClusterSerializer,
     QueryClusterIpsSerializer,
@@ -95,3 +96,14 @@ def get_cluster_capacity_update_info(self, request, bk_biz_id, **kwargs):
         update_info = get_cluster_capacity_update_required_info(**data)
         update_fields = ["capacity_update_type", "require_spec_id", "require_machine_group_num", "err_msg"]
         return Response(dict(zip(update_fields, update_info)))
+
+    @common_swagger_auto_schema(
+        operation_summary=_("获取集群module信息"),
+        query_serializer=GetClusterModuleInfoSerializer(),
+        tags=[SWAGGER_TAG],
+    )
+    @action(methods=["GET"], detail=False, serializer_class=GetClusterModuleInfoSerializer, pagination_class=None)
+    def get_cluster_module_info(self, request, bk_biz_id, **kwargs):
+        data = self.params_validate(self.get_serializer_class())
+        cluster_id, version = data["cluster_id"], data["version"]
+        return Response(ToolboxHandler.get_cluster_module_info(cluster_id, version))

dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py

@@ -137,4 +137,12 @@ def ClusterLoadModulesAtomJob(root_id, ticket_data, sub_kwargs: ActKwargs, param
     )
     logger.info("===>redis config")
 
+    act_kwargs.get_redis_payload_func = RedisActPayload.update_cluster_module.__name__
+    sub_pipeline.add_act(
+        act_name=_("更新cluster_module元数据"),
+        act_component_code=RedisConfigComponent.code,
+        kwargs=asdict(act_kwargs),
+    )
+    logger.info("===>redis update_cluster_module")
+
     return sub_pipeline.build_sub_process(sub_name=_("{}-集群加载modules").format(cluster.immute_domain))

dbm-ui/backend/flow/utils/redis/redis_act_playload.py

@@ -35,6 +35,7 @@
     get_dbmon_maxmemory_config_by_cluster_ids,
 )
 from backend.db_services.redis.redis_dts.models.tb_tendis_dts_switch_backup import TbTendisDtsSwitchBackup
+from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate
 from backend.db_services.redis.redis_modules.util import get_cluster_redis_modules_detial, get_redis_moudles_detail
 from backend.db_services.redis.util import (
     is_predixy_proxy_type,
@@ -2498,3 +2499,23 @@ def redis_replicas_force_resync(self, **kwargs) -> dict:
                 "slave_ports": params["slave_ports"],
             },
         }
+
+    def update_cluster_module(self, cluster_map: dict) -> bool:
+        """
+        更新nodes cluster_module记录
+        """
+        cluster_id = cluster_map["cluster_id"]
+        new_module_names = set(cluster_map.get("load_modules", []))
+
+        # 获取现有记录
+        obj, created = ClusterRedisModuleAssociate.objects.get_or_create(
+            cluster_id=cluster_id, defaults={"module_names": list(new_module_names)}
+        )
+        if not created:
+            # 如果记录已存在，合并现有的模块列表和新的模块列表
+            current_module_names = set(obj.module_names)
+            if current_module_names != new_module_names:
+                updated_module_names = list(current_module_names | new_module_names)
+                obj.module_names = updated_module_names
+                obj.save()
+        return True

dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.utils.translation import ugettext_lazy as _
+from rest_framework import serializers
+
+from backend.flow.engine.controller.redis import RedisController
+from backend.ticket import builders
+from backend.ticket.builders.common.base import DisplayInfoSerializer, SkipToRepresentationMixin
+from backend.ticket.builders.redis.base import BaseRedisTicketFlowBuilder, ClusterValidateMixin
+from backend.ticket.constants import LoadConfirmType, TicketType
+
+
+class RedisLoadModuleSerializer(SkipToRepresentationMixin, serializers.Serializer):
+    """redis集群加载module"""
+
+    class InfoSerializer(DisplayInfoSerializer, ClusterValidateMixin, serializers.Serializer):
+
+        cluster_id = serializers.IntegerField(help_text=_("集群ID"))
+        db_version = serializers.CharField(help_text=_("版本号"))
+        load_modules = serializers.ListField(
+            help_text=_("module类型列表"),
+            child=serializers.ChoiceField(
+                help_text=_("module类型"), choices=LoadConfirmType.get_choices(), default=LoadConfirmType.REDIS_BLOOM
+            ),
+            required=False,
+        )
+
+    bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID"))
+    infos = serializers.ListField(help_text=_("批量操作参数列表"), child=InfoSerializer())
+
+
+class RedisLoadModuleParamBuilder(builders.FlowParamBuilder):
+    controller = RedisController.redis_cluster_load_modules
+
+    def format_ticket_data(self):
+        super().format_ticket_data()
+
+
+@builders.BuilderFactory.register(TicketType.REDIS_CLUSTER_LOAD_MODULES, is_apply=True)
+class RedisLoadModuleFlowBuilder(BaseRedisTicketFlowBuilder):
+    serializer = RedisLoadModuleSerializer
+    inner_flow_builder = RedisLoadModuleParamBuilder
+    inner_flow_name = _("Redis 存量集群安装module")

dbm-ui/backend/ticket/constants.py

@@ -343,7 +343,7 @@ def get_cluster_type_by_ticket(cls, ticket_type):
     REDIS_CLUSTER_STORAGES_CLI_CONNS_KILL = TicketEnumField("REDIS_CLUSTER_STORAGES_CLI_CONNS_KILL", _("Redis 集群存储层cli连接kill"), register_iam=False)  # noqa
     REDIS_CLUSTER_RENAME_DOMAIN = TicketEnumField("REDIS_CLUSTER_RENAME_DOMAIN", _("Redis集群域名重命名"), _("集群维护"))
     REDIS_CLUSTER_MAXMEMORY_SET = TicketEnumField("REDIS_CLUSTER_MAXMEMORY_SET", _("Redis 集群设置maxmemory"))  # noqa
-    REDIS_CLUSTER_LOAD_MODULES = TicketEnumField("REDIS_CLUSTER_LOAD_MODULES", _("Redis 集群加载modules"))  # noqa
+    REDIS_CLUSTER_LOAD_MODULES = TicketEnumField("REDIS_CLUSTER_LOAD_MODULES", _("Redis 集群安装modules"))  # noqa
     REDIS_TENDISPLUS_LIGHTNING_DATA = TicketEnumField("REDIS_TENDISPLUS_LIGHTNING_DATA", _("Tendisplus闪电导入数据"), _("集群维护"))  # noqa
     REDIS_CLUSTER_INS_MIGRATE = TicketEnumField("REDIS_CLUSTER_INS_MIGRATE", _("Redis 集群指定实例迁移"), _("集群管理"))
     REDIS_SINGLE_INS_MIGRATE = TicketEnumField("REDIS_SINGLE_INS_MIGRATE", _("Redis 主从指定实例迁移"), _("集群管理"))
@@ -568,6 +568,20 @@ class SwitchConfirmType(str, StructuredEnum):
     NO_CONFIRM = EnumField("no_confirm", _("无需确认"))
 
 
+class LoadConfirmType(str, StructuredEnum):
+    """
+    加载Module类型
+    """
+
+    REDIS_BLOOM = EnumField("redisbloom", _("redisbloom"))
+    REDIS_CELL = EnumField("rediscell", _("rediscell"))
+    FO4_LOCK = EnumField("fo4_lock", _("fo4_lock"))
+    FO4_MATCHMAKER = EnumField("fo4_matchmaker", _("fo4_matchmaker"))
+    FO4_UTIL = EnumField("fo4_util", _("fo4_util"))
+    JLSY_B2 = EnumField("jlsy-b2", _("jlsy-b2"))
+    REDIS_JSON = EnumField("redisjson", _("redisjson"))
+
+
 class SyncDisconnectSettingType(str, StructuredEnum):
     """
     同步断开设置