feat(backend): 监控采集模板刷新，补充machine_types，支持一个插件多个采集 close #1583

dbm-ui/backend/db_meta/models/cluster_monitor.py

@@ -133,10 +133,13 @@ def get_set_by_dbtype(cls, db_type):
         ]
 
     @classmethod
-    def get_set_by_plugin_id(cls, plugin_id):
-        return list(
-            cls.objects.filter(monitor_plugin_id__contains=plugin_id).values_list("bk_set_id", "bk_biz_id").distinct()
-        )
+    def get_set_by_plugin_id(cls, plugin_id, machine_types=None):
+        print(plugin_id, machine_types)
+        qs = cls.objects.filter(monitor_plugin_id__contains=plugin_id)
+        if machine_types:
+            qs = qs.filter(machine_type__in=machine_types)
+
+        return list(qs.values_list("bk_set_id", "bk_biz_id").distinct())
 
     @classmethod
     @transaction.atomic

dbm-ui/backend/db_monitor/management/commands/export_template.py

@@ -28,7 +28,7 @@ class Command(BaseCommand):
 
     COLLECT_FIELDS = [
         "name",
-        "short_name",
+        "machine_types",
         "bk_biz_id",
         "plugin_id",
         "db_type",
@@ -70,7 +70,7 @@ def handle(self, *args, **options):
                 template = model_to_dict(template, fields=self.COLLECT_FIELDS)
                 template["version"] = template.get("version", 1)
                 template_json = json.dumps(template, indent=2)
-                template_file_name = "{db_type}.{name}.{short_name}.json".format(**template)
+                template_file_name = "{db_type}.{name}.json".format(**template)
                 with open(os.path.join(TPLS_COLLECT_DIR, template_file_name), "w") as template_file:
                     template_file.write(template_json)
                     print(f"export db: {db_type}'s {template_type}: {template['name']}")

dbm-ui/backend/db_monitor/management/commands/extract_alarm.py

@@ -9,16 +9,18 @@
 specific language governing permissions and limitations under the License.
 """
 import copy
+import json
 import logging
-from typing import List
+import os
+from collections import OrderedDict
 
 from django.core.management.base import BaseCommand
 
 from backend import env
 from backend.components import BKMonitorV3Api
 from backend.configuration.constants import DBType
-from backend.db_meta.models import AppMonitorTopo
-from backend.db_monitor.models import NoticeGroup, RuleTemplate
+from backend.db_monitor.constants import TPLS_ALARM_DIR, TargetPriority
+from backend.db_monitor.models import RuleTemplate
 
 logger = logging.getLogger("root")
 
@@ -29,14 +31,11 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument("db_type", choices=DBType.get_values(), type=str, help="db类型")
         parser.add_argument("bkmonitor_strategy_list", nargs="+", type=int, help="监控策略ID列表")
+        parser.add_argument("-d", "--disabled", dest="is_disabled", action="store_true", help="disable by default")
 
     def to_template(self, db_type: str, instance: dict):
         """告警策略模板化处理"""
 
-        def clear_id(objs: List[dict], id_name="id"):
-            for obj in objs:
-                obj.pop(id_name, None)
-
         template = copy.deepcopy(instance)
 
         # 剔除无用参数
@@ -58,8 +57,8 @@ def clear_id(objs: List[dict], id_name="id"):
         ]:
             template.pop(key, None)
 
-        clear_id([template])
-        clear_id(template["detects"])
+        self.clear_id([template])
+        self.clear_id(template["detects"])
 
         # 更新业务id
         template["bk_biz_id"] = env.DBA_APP_BK_BIZ_ID
@@ -68,12 +67,12 @@ def clear_id(objs: List[dict], id_name="id"):
 
         # 更新通知组
         notice = template["notice"]
-        clear_id([notice])
+        self.clear_id([notice])
         notice["user_groups"] = []
 
         # 更新监控项和metric_id
         items = template["items"]
-        clear_id(items)
+        self.clear_id(items)
         for item in items:
             # 更新监控目标为db_type对应的cmdb拓扑
             item["target"] = []
@@ -92,16 +91,17 @@ def clear_id(objs: List[dict], id_name="id"):
                     query_config["result_table_id"] = "bkmonitor_event_{event_data_id}"
                     logger.info(query_config.get("metric_id"))
 
-                # result_table_id = query_config.get("result_table_id")
-                # if result_table_id and custom_event_key in metric_id and bkmonitor_event_key in result_table_id:
-                #     query_config["result_table_id"] = "bkmonitor_event_{event_data_id}"
-                #     logger.info(query_config.get("result_table_id"))
-
         return instance["id"], template
 
+    def clear_id(self, objs, id_name="id"):
+        for obj in objs:
+            obj.pop(id_name, None)
+
     def handle(self, *args, **options):
         bkmonitor_strategy_list = options["bkmonitor_strategy_list"]
         db_type = options["db_type"]
+        is_disabled = options["is_disabled"]
+
         res = BKMonitorV3Api.search_alarm_strategy_v3(
             {
                 "page": 1,
@@ -118,14 +118,80 @@ def handle(self, *args, **options):
         strategy_config_list = res["strategy_config_list"]
         logger.info(f"[{db_type}]get {len(strategy_config_list)} alarm strategy")
         for strategy_config in strategy_config_list:
+
             # 策略转模板
             strategy_id, strategy_template = self.to_template(db_type, strategy_config)
-            logger.info(f"[{db_type}-{strategy_id}]update rule template: {strategy_template['name']}")
-            obj, _ = RuleTemplate.objects.update_or_create(
-                defaults={
-                    "name": strategy_template["name"],
-                    "details": strategy_template,
-                },
-                monitor_strategy_id=strategy_id,
-                db_type=db_type,
-            )
+            logger.info(f"[{db_type}-{strategy_id}]update policy template: {strategy_template['name']}")
+
+            # obj, _ = RuleTemplate.objects.update_or_create(
+            #     defaults={
+            #         "name": strategy_template["name"],
+            #         "details": strategy_template,
+            #     },
+            #     monitor_strategy_id=strategy_id,
+            #     db_type=db_type,
+            # )
+
+            template_name = strategy_template["name"]
+            strategy_template["labels"] = sorted(set(strategy_template["labels"]))
+            strategy_template["source"] = "dbm"
+            strategy_template["bk_biz_id"] = ""
+            strategy_template["priority"] = TargetPriority.PLATFORM.value
+
+            # 平台策略仅开启基于分派通知
+            strategy_template["notice"]["options"]["assign_mode"] = ["by_rule"]
+            strategy_template["labels"] = sorted(set(strategy_template["labels"]))
+
+            data_type_label = ""
+            for item in strategy_template["items"]:
+                # 清空监控目标
+                item["target"] = []
+                item["origin_sql"] = ""
+
+                # 补充app_id作为维度
+                self.clear_id(item["query_configs"])
+                self.clear_id(item["algorithms"])
+
+                for query_config in item["query_configs"]:
+                    metric_id = query_config["metric_id"]
+
+                    # 标记告警数据来源
+                    data_type_label = query_config["data_type_label"]
+
+                    if "promql" in query_config:
+                        promql = query_config["promql"]
+                        if metric_id != promql:
+                            query_config["metric_id"] = promql
+                    else:
+                        # 奇怪，监控新版阉割了name
+                        metric_field = query_config.get("metric_field")
+                        if metric_field and not metric_id.endswith(metric_field):
+                            query_config["metric_id"] = ".".join(metric_id.split(".")[:-1] + [metric_field])
+                            logger.info(f"found bad metric_id rule: {template_name} -> {query_config['metric_id']}")
+
+                    # 根据监控侧长度限制进行截断
+                    if len(query_config["metric_id"]) > 128:
+                        logger.info(f"found too long promql metric: {template_name} -> {query_config['metric_id']}")
+
+                    query_config["metric_id"] = query_config["metric_id"][:128]
+
+                    if "agg_dimension" in query_config and "appid" not in query_config["agg_dimension"]:
+                        query_config["agg_dimension"].append("appid")
+
+            self.clear_id(strategy_template["items"])
+
+            with open(os.path.join(TPLS_ALARM_DIR, f"{template_name}_test.json"), "w") as template_file:
+                template_dict = OrderedDict(
+                    {
+                        "bk_biz_id": 0,
+                        "version": 0,
+                        "is_enabled": not is_disabled,
+                        "name": template_name,
+                        "details": strategy_template,
+                        "db_type": db_type,
+                        "alert_source": data_type_label,
+                        "monitor_indicator": strategy_template["items"][0]["name"],
+                    }
+                )
+
+                template_file.write(json.dumps(template_dict, indent=2))

dbm-ui/backend/db_monitor/management/commands/extract_collect.py

@@ -9,14 +9,19 @@
 specific language governing permissions and limitations under the License.
 """
 import copy
+import json
 import logging
+import os
+from collections import OrderedDict
 from typing import List
 
 from django.core.management.base import BaseCommand
 
+from backend import env
 from backend.components import BKMonitorV3Api
 from backend.db_meta.enums import MachineType
 from backend.db_meta.models.cluster_monitor import SHORT_NAMES
+from backend.db_monitor.constants import TPLS_COLLECT_DIR
 from backend.db_monitor.models import CollectTemplate
 
 logger = logging.getLogger("root")
@@ -29,8 +34,8 @@ def add_arguments(self, parser):
         parser.add_argument(
             "db_type", choices=["mysql", "redis", "es", "hdfs", "kafka", "pulsar", "influxdb"], type=str, help="db类型"
         )
-        parser.add_argument("short_name", choices=SHORT_NAMES + ["redis"], help="集群名后缀，比如db.redis.proxy -> proxy")
         parser.add_argument("collect_list", nargs="+", type=str, help="监控采集策略ID列表")
+        parser.add_argument("-m", "--machine_types", nargs="*", help="machine类型列表，可以为空", type=str)
 
     def to_template(self, instance: dict):
         """监控采集策略模板化处理"""
@@ -80,25 +85,43 @@ def clear_keys(obj: dict, keeped_keys: List):
         return template
 
     def handle(self, *args, **options):
-        collect_list = options["collect_list"]
         db_type = options["db_type"]
-        short_name = options["short_name"]
+        collect_list = options["collect_list"]
+        machine_types = options["machine_types"] or []
 
         # 批量获取策略
         for collect_id in collect_list:
             instance = BKMonitorV3Api.query_collect_config_detail({"id": str(collect_id)})
             # 策略转模板
 
             plugin_id = instance["plugin_info"]["plugin_id"]
-            collect_template = self.to_template(instance)
-            print(collect_template)
-            logger.info(f"[{db_type}-{collect_id}] update collect template: {collect_template['name']}")
-            obj, _ = CollectTemplate.objects.update_or_create(
-                defaults={
-                    "name": collect_template["name"],
-                    "details": collect_template,
-                },
-                db_type=db_type,
-                short_name=short_name,
-                plugin_id=plugin_id,
-            )
+            template = self.to_template(instance)
+
+            # obj, _ = CollectTemplate.objects.update_or_create(
+            #     defaults={
+            #         "name": template["name"],
+            #         "details": template,
+            #     },
+            #     db_type=db_type,
+            #     machine_types=machine_types,
+            #     plugin_id=plugin_id,
+            # )
+
+            logger.info(f"[{db_type}-{collect_id}] update collect template: {template['name']}")
+            with open(os.path.join(TPLS_COLLECT_DIR, template["name"]) + ".json", "w") as template_file:
+                template_file.write(
+                    json.dumps(
+                        OrderedDict(
+                            {
+                                "bk_biz_id": 0,
+                                "name": template["name"],
+                                "details": template,
+                                "db_type": db_type,
+                                "version": template.get("version", 0) + 1,
+                                "machine_types": machine_types,
+                                "plugin_id": plugin_id,
+                            }
+                        ),
+                        indent=2,
+                    )
+                )

dbm-ui/backend/db_monitor/management/commands/fix_collect_template.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import glob
+import json
+import logging
+import os
+
+from django.core.management.base import BaseCommand
+
+from backend.db_monitor.constants import TPLS_ALARM_DIR, TPLS_COLLECT_DIR, TargetPriority
+
+logger = logging.getLogger("root")
+
+
+class Command(BaseCommand):
+    help = "采集模板文件修复"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-d",
+            "--dbtype",
+            choices=["mysql", "redis", "es", "hdfs", "kafka", "pulsar", "influxdb", "all"],
+            default="all",
+            type=str,
+            help="db类型",
+        )
+
+    def update_json_file(self, f, template_dict):
+        f.seek(0)
+        f.write(json.dumps(template_dict, indent=2))
+        f.truncate()
+
+    def handle(self, *args, **options):
+        db_type = options["dbtype"]
+        json_files = glob.glob(os.path.join(TPLS_COLLECT_DIR, "*.json"))
+        for json_file in json_files:
+            with open(json_file, "r+") as f:
+                template_dict = json.loads(f.read())
+                # 补充 machine_types 参数
+                if not template_dict.get("machine_types"):
+                    template_dict["machine_types"] = []
+
+                template_dict.pop("short_name", None)
+                self.update_json_file(f, template_dict)

dbm-ui/backend/db_monitor/management/commands/fix_template.py → dbm-ui/backend/db_monitor/management/commands/fix_policy_template.py

@@ -119,18 +119,5 @@ def handle(self, *args, **options):
                             query_config["agg_dimension"].pop("app_id", None)
 
                 self.clear_id(details["items"])
-
-                # # [HDFS]-XXXXX -> HDFS XXXXX
-                # if old_template_name.startswith("[") and "]-" in old_template_name:
-                #     db_type, name = old_template_name.split("-")
-                #     template_name = f"{db_type[1:-1]} {name}"
-                #     template_dict["details"]["name"] = template_dict["name"] = template_name
-                #
-                # # redis主机xxx-> Redis 主机xxx
-                # if old_template_name.startswith("redis主机"):
-                #     db_type = "redis"
-                #     template_name = f"Redis {old_template_name[5:]}"
-                #     template_dict["details"]["name"] = template_dict["name"] = template_name
-
                 self.update_json_file_name(alarm_json, old_template_name, template_name)
                 self.update_json_file(f, template_dict)