feat(backend): redis日志采集项修复 close #2198

TencentBlueKing · Nov 29, 2023 · b3a6957 · b3a6957
1 parent f7666f6
commit b3a6957
Show file tree

Hide file tree

Showing 5 changed files with 87 additions and 10 deletions.
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json
@@ -27,6 +27,22 @@
         },
         "encoding": "UTF-8"
     },
+    "etl_params": {
+        "es_unique_field_list": [
+            "cloudId",
+            "serverIp",
+            "path",
+            "gseIndex",
+            "iterationIndex",
+            "bk_host_id"
+        ],
+        "separator_node_source": "data",
+        "separator_node_action": "json",
+        "separator_node_name": "bk_separator_object",
+        "retain_original_text": true,
+        "separator_fields_remove": "",
+        "etl_flat": false
+    },
     "fields": [
         {
             "value": "2023-11-17T00:00:03.534Z",

diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json
@@ -1,9 +1,9 @@
 {
     "bk_biz_id": {{bk_biz_id}},
     "etl_config": "bk_log_json",
-    "table_id": "redis_binlog_backup_result1",
-    "collector_config_name": "redis_binlog_backup_result1",
-    "collector_config_name_en": "redis_binlog_backup_result1",
+    "table_id": "redis_binlog_backup_result",
+    "collector_config_name": "redis_binlog_backup_result",
+    "collector_config_name_en": "redis_binlog_backup_result",
     "collector_scenario_id": "row",
     "category_id": "host_process",
     "storage_cluster_id": 3,
@@ -27,6 +27,22 @@
         },
         "encoding": "UTF-8"
     },
+    "etl_params": {
+        "es_unique_field_list": [
+            "cloudId",
+            "serverIp",
+            "path",
+            "gseIndex",
+            "iterationIndex",
+            "bk_host_id"
+        ],
+        "separator_node_source": "data",
+        "separator_node_action": "json",
+        "separator_node_name": "bk_separator_object",
+        "retain_original_text": true,
+        "separator_fields_remove": "",
+        "etl_flat": false
+    },
     "fields": [
         {
             "value": "/data/dbbak/binlog/30000",

diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json
@@ -1,9 +1,9 @@
 {
     "bk_biz_id": {{bk_biz_id}},
     "etl_config": "bk_log_json",
-    "table_id": "redis_cluster_nodes_result1",
-    "collector_config_name": "redis_cluster_nodes_result1",
-    "collector_config_name_en": "redis_cluster_nodes_result1",
+    "table_id": "redis_cluster_nodes_result",
+    "collector_config_name": "redis_cluster_nodes_result",
+    "collector_config_name_en": "redis_cluster_nodes_result",
     "collector_scenario_id": "row",
     "category_id": "host_process",
     "storage_cluster_id": 3,
@@ -27,6 +27,22 @@
         },
         "encoding": "UTF-8"
     },
+    "etl_params": {
+        "es_unique_field_list": [
+            "cloudId",
+            "serverIp",
+            "path",
+            "gseIndex",
+            "iterationIndex",
+            "bk_host_id"
+        ],
+        "separator_node_source": "data",
+        "separator_node_action": "json",
+        "separator_node_name": "bk_separator_object",
+        "retain_original_text": true,
+        "separator_fields_remove": "",
+        "etl_flat": false
+    },
     "fields": [
         {
             "value": "3",

diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_fullbackup_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_fullbackup_result.json
@@ -1,9 +1,9 @@
 {
     "bk_biz_id": {{bk_biz_id}},
     "etl_config": "bk_log_json",
-    "table_id": "redis_fullbackup_result1",
-    "collector_config_name": "redis_fullbackup_result1",
-    "collector_config_name_en": "redis_fullbackup_result1",
+    "table_id": "redis_fullbackup_result",
+    "collector_config_name": "redis_fullbackup_result",
+    "collector_config_name_en": "redis_fullbackup_result",
     "collector_scenario_id": "row",
     "category_id": "host_process",
     "storage_cluster_id": 3,
@@ -27,6 +27,22 @@
         },
         "encoding": "UTF-8"
     },
+    "etl_params": {
+        "es_unique_field_list": [
+            "cloudId",
+            "serverIp",
+            "path",
+            "gseIndex",
+            "iterationIndex",
+            "bk_host_id"
+        ],
+        "separator_node_source": "data",
+        "separator_node_action": "json",
+        "separator_node_name": "bk_separator_object",
+        "retain_original_text": true,
+        "separator_fields_remove": "",
+        "etl_flat": false
+    },
     "fields": [
         {
             "value": "/data/dbbak",

diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_keymod.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_keymod.json
@@ -25,7 +25,20 @@
     "es_shards": 3,
     "allocation_min_days": 0,
     "etl_params": {
-        "retain_original_text": true
+        "es_unique_field_list": [
+            "cloudId",
+            "serverIp",
+            "path",
+            "gseIndex",
+            "iterationIndex",
+            "bk_host_id"
+        ],
+        "separator_node_source": "data",
+        "separator_node_action": "json",
+        "separator_node_name": "bk_separator_object",
+        "retain_original_text": true,
+        "separator_fields_remove": "",
+        "etl_flat": false
     },
     "fields": [
         {