From 456798e4307580653c1942c98ca0f384861e20b5 Mon Sep 17 00:00:00 2001 From: durant <826035498@qq.com> Date: Wed, 20 Dec 2023 15:37:54 +0800 Subject: [PATCH] =?UTF-8?q?feat(backend):=20redis=20=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=97=B6=E5=8C=BA=20#2688?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/auto_create_release.yml | 4 +- .../json_files/bklog/backup_stm_log.json | 1 - .../json_files/bklog/dbm_dbactuator.json | 5 +- .../json_files/bklog/mysql_binlog_result.json | 751 +++++++------- .../bklog/mysql_checksum_result.json | 5 +- .../bklog/mysql_dbbackup_result.json | 971 +++++++++--------- .../json_files/bklog/mysql_slowlog.json | 299 +++--- .../json_files/bklog/redis_bigkey.json | 1 - .../bklog/redis_binlog_backup_result.json | 74 +- .../bklog/redis_cluster_nodes_result.json | 1 - .../bklog/redis_fullbackup_result.json | 1 - .../json_files/bklog/redis_keymod.json | 1 - .../json_files/bklog/redis_slowlog.json | 37 +- dbm-ui/backend/dbm_init/json_files/format.py | 2 - dbm-ui/backend/dbm_init/services.py | 11 +- dbm-ui/backend/env/bklog.py | 6 + 16 files changed, 1112 insertions(+), 1058 deletions(-) diff --git a/.github/workflows/auto_create_release.yml b/.github/workflows/auto_create_release.yml index 8ff264a3d0..bb9b859001 100644 --- a/.github/workflows/auto_create_release.yml +++ b/.github/workflows/auto_create_release.yml @@ -86,7 +86,7 @@ jobs: cat ${filename} git config --global user.email github-actions@users.noreply.github.com git config --global user.name github-actions - git remote set-url origin https://x-access-token:$WORKFLOW_ACTION_TOKEN@github.com/TencentBlueKing/blueking-dbm + git remote set-url upstream https://$WORKFLOW_ACTION_TOKEN@github.com/TencentBlueKing/blueking-dbm git add . git commit -m "docs: release version ${{ steps.yaml-data.outputs.data }} #1" - git push origin ${{ steps.get-current-branch.outputs.current_branch }} \ No newline at end of file + git push upstream ${{ steps.get-current-branch.outputs.current_branch }} diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/backup_stm_log.json b/dbm-ui/backend/dbm_init/json_files/bklog/backup_stm_log.json index a9a32abc0e..e732debfdc 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/backup_stm_log.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/backup_stm_log.json @@ -9,7 +9,6 @@ "environment": "linux", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/dbm_dbactuator.json b/dbm-ui/backend/dbm_init/json_files/bklog/dbm_dbactuator.json index d54b7b2158..5ac14b4761 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/dbm_dbactuator.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/dbm_dbactuator.json @@ -7,13 +7,14 @@ "etl_config": "bk_log_text", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", "target_nodes": [], "params": { - "paths": ["/data/install/dbactuator*/logs/*.log"], + "paths": [ + "/data/install/dbactuator*/logs/*.log" + ], "conditions": { "separator": "", "separator_filters": [], diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_binlog_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_binlog_result.json index 7be72060a8..eba84986ec 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_binlog_result.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_binlog_result.json @@ -1,380 +1,379 @@ { - "bk_biz_id": null, - "collector_config_name": "mysql_binlog_result", - "collector_config_name_en": "mysql_binlog_result", - "collector_scenario_id": "row", - "category_id": "host_process", - "etl_config": "bk_log_json", - "data_encoding": "UTF-8", - "retention": 7, - "allocation_min_days": 3, - "es_shards": 3, - "target_object_type": "HOST", - "target_node_type": "TOPO", - "target_nodes": [], - "etl_params": { - "retain_original_text": true - }, - "params": { - "paths": [ - "/home/mysql/dbareport/mysql/binlog/binlog_result*.log" - ], - "conditions": { - "separator": "", - "separator_filters": [], - "type": "match", - "match_type": "include", - "match_content": "" + "bk_biz_id": null, + "collector_config_name": "mysql_binlog_result", + "collector_config_name_en": "mysql_binlog_result", + "collector_scenario_id": "row", + "category_id": "host_process", + "etl_config": "bk_log_json", + "data_encoding": "UTF-8", + "retention": 7, + "es_shards": 3, + "target_object_type": "HOST", + "target_node_type": "TOPO", + "target_nodes": [], + "etl_params": { + "retain_original_text": true }, - "tail_files": true, - "ignore_older": 2678400, - "max_bytes": 204800 - }, - "fields": [ - { - "tag": "dimension", - "type": "float", - "unit": "", - "option": { - "es_type": "integer", - "real_path": "bk_separator_object.backup_status", - "time_zone": "", - "field_index": 1, - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "backup_status", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "int", - "is_config_by_user": true + "params": { + "paths": [ + "/home/mysql/dbareport/mysql/binlog/binlog_result*.log" + ], + "conditions": { + "separator": "", + "separator_filters": [], + "type": "match", + "match_type": "include", + "match_content": "" + }, + "tail_files": true, + "ignore_older": 2678400, + "max_bytes": 204800 }, - { - "tag": "dimension", - "type": "string", - "unit": "", - "option": { - "es_type": "keyword", - "real_path": "bk_separator_object.backup_status_info", - "time_zone": "", - "field_index": 2, - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_status_info", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "string", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "float", - "unit": "", - "option": { - "es_type": "integer", - "real_path": "bk_separator_object.bk_biz_id", - "time_zone": "", - "field_index": 4, - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "bkbizid", - "field_name": "bk_biz_id", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "int", - "is_config_by_user": true - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "cluster_domain", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "tag": "dimension", - "type": "float", - "unit": "", - "option": { - "es_type": "integer", - "real_path": "bk_separator_object.cluster_id", - "time_zone": "", - "field_index": 5, - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "cluster_id", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "int", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "string", - "unit": "", - "option": { - "es_type": "keyword", - "real_path": "bk_separator_object.db_role", - "time_zone": "", - "field_index": 6, - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "db_role", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "string", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "string", - "unit": "", - "option": { - "es_type": "keyword", - "real_path": "bk_separator_object.file_mtime", - "time_zone": 8, - "field_index": 7, - "time_format": "strict_date_time_no_millis" - }, - "is_time": true, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "file_mtime", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "string", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "string", - "unit": "", - "option": { - "es_type": "keyword", - "real_path": "bk_separator_object.filename", - "time_zone": "", - "field_index": 8, - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "file_name", - "field_name": "filename", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "string", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "string", - "unit": "", - "option": { - "es_type": "keyword", - "real_path": "bk_separator_object.host", - "time_zone": "", - "field_index": 9, - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "host", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "string", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "float", - "unit": "", - "option": { - "es_type": "integer", - "real_path": "bk_separator_object.port", - "time_zone": "", - "field_index": 10, - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "port", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "int", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "float", - "unit": "", - "option": { - "es_type": "long", - "real_path": "bk_separator_object.size", - "time_zone": "", - "field_index": 11, - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "size", - "field_type": "long", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "long", - "is_config_by_user": true - }, - { - "tag": "metric", - "type": "string", - "unit": "", - "option": { - "real_path": "bk_separator_object.start_time", - "time_zone": "", - "field_index": 12, - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "start_time", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "string", - "is_config_by_user": true - }, - { - "tag": "dimension", - "type": "string", - "unit": "", - "option": { - "es_type": "keyword", - "real_path": "bk_separator_object.stop_time", - "time_zone": "", - "field_index": 13, - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "stop_time", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_disabled": false, - "is_dimension": true, - "default_value": null, - "previous_type": "string", - "is_config_by_user": true - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "task_id", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - } - ] + "fields": [ + { + "tag": "dimension", + "type": "float", + "unit": "", + "option": { + "es_type": "integer", + "real_path": "bk_separator_object.backup_status", + "time_zone": "", + "field_index": 1, + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "backup_status", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "int", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "string", + "unit": "", + "option": { + "es_type": "keyword", + "real_path": "bk_separator_object.backup_status_info", + "time_zone": "", + "field_index": 2, + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_status_info", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "string", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "float", + "unit": "", + "option": { + "es_type": "integer", + "real_path": "bk_separator_object.bk_biz_id", + "time_zone": "", + "field_index": 4, + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "bkbizid", + "field_name": "bk_biz_id", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "int", + "is_config_by_user": true + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "cluster_domain", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "tag": "dimension", + "type": "float", + "unit": "", + "option": { + "es_type": "integer", + "real_path": "bk_separator_object.cluster_id", + "time_zone": "", + "field_index": 5, + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "cluster_id", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "int", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "string", + "unit": "", + "option": { + "es_type": "keyword", + "real_path": "bk_separator_object.db_role", + "time_zone": "", + "field_index": 6, + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "db_role", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "string", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "string", + "unit": "", + "option": { + "es_type": "keyword", + "real_path": "bk_separator_object.file_mtime", + "time_zone": 8, + "field_index": 7, + "time_format": "strict_date_time_no_millis" + }, + "is_time": true, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "file_mtime", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "string", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "string", + "unit": "", + "option": { + "es_type": "keyword", + "real_path": "bk_separator_object.filename", + "time_zone": "", + "field_index": 8, + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "file_name", + "field_name": "filename", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "string", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "string", + "unit": "", + "option": { + "es_type": "keyword", + "real_path": "bk_separator_object.host", + "time_zone": "", + "field_index": 9, + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "host", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "string", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "float", + "unit": "", + "option": { + "es_type": "integer", + "real_path": "bk_separator_object.port", + "time_zone": "", + "field_index": 10, + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "port", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "int", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "float", + "unit": "", + "option": { + "es_type": "long", + "real_path": "bk_separator_object.size", + "time_zone": "", + "field_index": 11, + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "size", + "field_type": "long", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "long", + "is_config_by_user": true + }, + { + "tag": "metric", + "type": "string", + "unit": "", + "option": { + "real_path": "bk_separator_object.start_time", + "time_zone": "", + "field_index": 12, + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "start_time", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "string", + "is_config_by_user": true + }, + { + "tag": "dimension", + "type": "string", + "unit": "", + "option": { + "es_type": "keyword", + "real_path": "bk_separator_object.stop_time", + "time_zone": "", + "field_index": 13, + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "stop_time", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_disabled": false, + "is_dimension": true, + "default_value": null, + "previous_type": "string", + "is_config_by_user": true + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "task_id", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + } + ] } \ No newline at end of file diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_checksum_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_checksum_result.json index 01537553ce..9b5c81773f 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_checksum_result.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_checksum_result.json @@ -7,13 +7,14 @@ "etl_config": "bk_log_text", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", "target_nodes": [], "params": { - "paths": ["/home/mysql/dbareport/checksum/*.log"], + "paths": [ + "/home/mysql/dbareport/checksum/*.log" + ], "conditions": { "separator": "", "separator_filters": [], diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_dbbackup_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_dbbackup_result.json index 777ec2f7b8..098b486f20 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_dbbackup_result.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_dbbackup_result.json @@ -1,490 +1,489 @@ { - "bk_biz_id": null, - "collector_config_name": "mysql_dbbackup_result", - "collector_config_name_en": "mysql_dbbackup_result", - "collector_scenario_id": "row", - "category_id": "host_process", - "etl_config": "bk_log_json", - "data_encoding": "UTF-8", - "retention": 7, - "allocation_min_days": 3, - "es_shards": 3, - "target_object_type": "HOST", - "target_node_type": "TOPO", - "target_nodes": [], - "params": { - "paths": [ - "/home/mysql/dbareport/mysql/dbbackup/backup_result*.log" - ], - "conditions": { - "separator": "", - "separator_filters": [], - "type": "match", - "match_type": "include", - "match_content": "" + "bk_biz_id": null, + "collector_config_name": "mysql_dbbackup_result", + "collector_config_name_en": "mysql_dbbackup_result", + "collector_scenario_id": "row", + "category_id": "host_process", + "etl_config": "bk_log_json", + "data_encoding": "UTF-8", + "retention": 7, + "es_shards": 3, + "target_object_type": "HOST", + "target_node_type": "TOPO", + "target_nodes": [], + "params": { + "paths": [ + "/home/mysql/dbareport/mysql/dbbackup/backup_result*.log" + ], + "conditions": { + "separator": "", + "separator_filters": [], + "type": "match", + "match_type": "include", + "match_content": "" + }, + "tail_files": true, + "ignore_older": 2678400, + "max_bytes": 204800 }, - "tail_files": true, - "ignore_older": 2678400, - "max_bytes": 204800 - }, - "etl_params": { - "es_unique_field_list": [ - "cloudId", - "serverIp", - "path", - "gseIndex", - "iterationIndex", - "bk_host_id" - ], - "separator_node_source": "data", - "separator_node_action": "json", - "separator_node_name": "bk_separator_object", - "retain_original_text": true, - "separator_fields_remove": "", - "etl_flat": false - }, - "fields": [ - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_begin_time", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" + "etl_params": { + "es_unique_field_list": [ + "cloudId", + "serverIp", + "path", + "gseIndex", + "iterationIndex", + "bk_host_id" + ], + "separator_node_source": "data", + "separator_node_action": "json", + "separator_node_name": "bk_separator_object", + "retain_original_text": true, + "separator_fields_remove": "", + "etl_flat": false }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_charset", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": 8, - "time_format": "strict_date_time_no_millis" - }, - "is_time": true, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_consistent_time", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_end_time", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_host", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_id", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "backup_port", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "int" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "backup_type", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "bill_id", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "binlog_info", - "field_type": "object", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "object" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "bkbizid", - "field_name": "bk_biz_id", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "int" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "bkcloudid", - "field_name": "bk_cloud_id", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "int" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "cluster_address", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "cluster_id", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "int" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "consistent_backup_time", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "data_schema_grant", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "encrypt_enable", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "file_list", - "field_type": "object", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "object" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "is_full_backup", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "int" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "mysql_role", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "mysql_version", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "shard_value", - "field_type": "int", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "int" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "storage_engine", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": true, - "is_delete": false, - "alias_name": "", - "field_name": "time_zone", - "field_type": "string", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "string" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "total_filesize", - "field_type": "long", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "long" - }, - { - "option": { - "time_zone": "", - "time_format": "" - }, - "is_time": false, - "verdict": false, - "is_delete": false, - "alias_name": "", - "field_name": "total_size_kb_uncompress", - "field_type": "long", - "description": "", - "is_analyzed": false, - "is_built_in": false, - "is_dimension": false, - "previous_type": "long" - } - ] + "fields": [ + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_begin_time", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_charset", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": 8, + "time_format": "strict_date_time_no_millis" + }, + "is_time": true, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_consistent_time", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_end_time", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_host", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_id", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "backup_port", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "int" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "backup_type", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "bill_id", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "binlog_info", + "field_type": "object", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "object" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "bkbizid", + "field_name": "bk_biz_id", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "int" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "bkcloudid", + "field_name": "bk_cloud_id", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "int" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "cluster_address", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "cluster_id", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "int" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "consistent_backup_time", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "data_schema_grant", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "encrypt_enable", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "file_list", + "field_type": "object", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "object" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "is_full_backup", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "int" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "mysql_role", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "mysql_version", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "shard_value", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "int" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "storage_engine", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "time_zone", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "total_filesize", + "field_type": "long", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "long" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "total_size_kb_uncompress", + "field_type": "long", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "long" + } + ] } \ No newline at end of file diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_slowlog.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_slowlog.json index cbc013db87..383c77b7d6 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_slowlog.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_slowlog.json @@ -7,172 +7,173 @@ "etl_config": "bk_log_regexp", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "SERVICE", "target_node_type": "TOPO", "target_nodes": [], "etl_params": { "retain_original_text": true, - "separator_regexp":"#\\s+User@Host:\\s+(?P\\w+)\\[\\w+\\]\\s+@\\s+(?P(\\w|\\[|\\]|\\.|\\s)+)\\s+Id:\\s+(?P\\d+)\\n#\\s+Query_time:\\s+(?P\\w+\\.?\\w+)\\s+Lock_time:\\s+(?P\\w+\\.?\\w+)\\s+Rows_sent:\\s+(?P\\d+)\\s+Rows_examined:\\s+(?P\\d+)\\n(use\\s+(?P.*);\\n)?SET\\s+timestamp=(?P\\d+);\\n(?P(.|\\n)*)" + "separator_regexp": "#\\s+User@Host:\\s+(?P\\w+)\\[\\w+\\]\\s+@\\s+(?P(\\w|\\[|\\]|\\.|\\s)+)\\s+Id:\\s+(?P\\d+)\\n#\\s+Query_time:\\s+(?P\\w+\\.?\\w+)\\s+Lock_time:\\s+(?P\\w+\\.?\\w+)\\s+Rows_sent:\\s+(?P\\d+)\\s+Rows_examined:\\s+(?P\\d+)\\n(use\\s+(?P.*);\\n)?SET\\s+timestamp=(?P\\d+);\\n(?P(.|\\n)*)" }, "fields": [ { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":true, - "is_delete":false, - "alias_name":"", - "field_name":"user", - "field_type":"string", - "description":"", - "field_index":1, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"string" + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "user", + "field_type": "string", + "description": "", + "field_index": 1, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":true, - "is_delete":false, - "alias_name":"", - "field_name":"host", - "field_type":"string", - "description":"", - "field_index":2, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"string" + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "host", + "field_type": "string", + "description": "", + "field_index": 2, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":false, - "is_delete":false, - "alias_name":"", - "field_name":"thread_id", - "field_type":"int", - "description":"", - "field_index":3, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"" + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "thread_id", + "field_type": "int", + "description": "", + "field_index": 3, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":false, - "is_delete":false, - "alias_name":"", - "field_name":"query_time", - "field_type":"double", - "description":"", - "field_index":4, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"" + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "query_time", + "field_type": "double", + "description": "", + "field_index": 4, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":false, - "is_delete":false, - "alias_name":"", - "field_name":"lock_time", - "field_type":"double", - "description":"", - "field_index":5, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"" + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "lock_time", + "field_type": "double", + "description": "", + "field_index": 5, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":false, - "is_delete":false, - "alias_name":"", - "field_name":"rows_sent", - "field_type":"int", - "description":"", - "field_index":6, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"" + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "rows_sent", + "field_type": "int", + "description": "", + "field_index": 6, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":false, - "is_delete":false, - "alias_name":"", - "field_name":"rows_examined", - "field_type":"int", - "description":"", - "field_index":7, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"" + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "rows_examined", + "field_type": "int", + "description": "", + "field_index": 7, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":true, - "is_delete":false, - "alias_name":"", - "field_name":"use", - "field_type":"string", - "description":"", - "field_index":8, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"" + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "use", + "field_type": "string", + "description": "", + "field_index": 8, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "" }, { - "value":"", - "option":{ + "value": "", + "option": { }, - "is_time":false, - "verdict":false, - "is_delete":false, - "alias_name":"", - "field_name":"sql_timestamp", - "field_type":"int", - "description":"", - "field_index":9, - "is_analyzed":false, - "is_built_in":false, - "is_dimension":false, - "previous_type":"" + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "sql_timestamp", + "field_type": "int", + "description": "", + "field_index": 9, + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "" } ], "params": { - "paths": ["/data/mysqllog/{{ cmdb_instance.service.labels[\"instance_port\"] }}/slow-query.log"], + "paths": [ + "/data/mysqllog/{{ cmdb_instance.service.labels[\"instance_port\"] }}/slow-query.log" + ], "conditions": { "separator": "", "separator_filters": [], @@ -180,25 +181,33 @@ "match_type": "include", "match_content": "" }, - "multiline_pattern":"^# User@Host", - "multiline_max_lines":50, - "multiline_timeout":2, - "encoding":"UTF-8", + "multiline_pattern": "^# User@Host", + "multiline_max_lines": 50, + "multiline_timeout": 2, + "encoding": "UTF-8", "extra_template_labels": [ - {"key": "$for", "value": "cmdb_instance.scope"}, - {"key": "$body", "value": { - "app": "{{ cmdb_instance.service.labels[\"app\"] }}", - "app_id": "{{ cmdb_instance.service.labels[\"app_id\"] }}", - "bk_cloud_id": "{{ cmdb_instance.service.labels[\"bk_cloud_id\"] }}", - "cluster_name": "{{ cmdb_instance.service.labels[\"cluster_name\"] }}", - "cluster_type": "{{ cmdb_instance.service.labels[\"cluster_type\"] }}", - "instance_host": "{{ cmdb_instance.service.labels[\"instance_host\"] }}", - "instance_port": "{{ cmdb_instance.service.labels[\"instance_port\"] }}", - "instance_role": "{{ cmdb_instance.service.labels[\"instance_role\"] }}", - "cluster_domain": "{{ cmdb_instance.service.labels[\"cluster_domain\"] }}" + { + "key": "$for", + "value": "cmdb_instance.scope" + }, + { + "key": "$body", + "value": { + "app": "{{ cmdb_instance.service.labels[\"app\"] }}", + "app_id": "{{ cmdb_instance.service.labels[\"app_id\"] }}", + "bk_cloud_id": "{{ cmdb_instance.service.labels[\"bk_cloud_id\"] }}", + "cluster_name": "{{ cmdb_instance.service.labels[\"cluster_name\"] }}", + "cluster_type": "{{ cmdb_instance.service.labels[\"cluster_type\"] }}", + "instance_host": "{{ cmdb_instance.service.labels[\"instance_host\"] }}", + "instance_port": "{{ cmdb_instance.service.labels[\"instance_port\"] }}", + "instance_role": "{{ cmdb_instance.service.labels[\"instance_role\"] }}", + "cluster_domain": "{{ cmdb_instance.service.labels[\"cluster_domain\"] }}" } }, - {"key": "$item", "value": "scope"} + { + "key": "$item", + "value": "scope" + } ] } } diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json index 96cb992bdb..485971e33d 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json @@ -8,7 +8,6 @@ "category_id": "application_check", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json index 750178c9c5..7e2bcdeb6e 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json @@ -8,7 +8,6 @@ "category_id": "host_process", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", @@ -44,7 +43,6 @@ }, "fields": [ { - "value": "/data/dbbak/binlog/30000", "option": { "time_zone": "", "time_format": "" @@ -62,7 +60,6 @@ "previous_type": "string" }, { - "value": "/data/dbbak/binlog/30000/binlog-127.0.0.1-30000-0006446-20231108140016.log.zst", "option": { "time_zone": "", "time_format": "" @@ -80,7 +77,6 @@ "previous_type": "string" }, { - "value": 5586, "option": { "time_zone": "", "time_format": "" @@ -98,7 +94,6 @@ "previous_type": "long" }, { - "value": "", "option": { "time_zone": "", "time_format": "" @@ -116,7 +111,6 @@ "previous_type": "string" }, { - "value": "REDIS_BINLOG", "option": { "time_zone": "", "time_format": "" @@ -134,13 +128,12 @@ "previous_type": "string" }, { - "value": "1699425099084113594-0160215607-9232-0", "option": { "time_zone": "", "time_format": "" }, "is_time": false, - "verdict": false, + "verdict": true, "is_delete": false, "alias_name": "", "field_name": "backup_taskid", @@ -152,7 +145,6 @@ "previous_type": "string" }, { - "value": "3", "option": { "time_zone": "", "time_format": "" @@ -170,7 +162,6 @@ "previous_type": "string" }, { - "value": 0, "option": { "time_zone": "", "time_format": "" @@ -188,7 +179,6 @@ "previous_type": "int" }, { - "value": "TendisSSDInstance", "option": { "time_zone": "", "time_format": "" @@ -206,7 +196,6 @@ "previous_type": "string" }, { - "value": "ssd.abc.dba.db", "option": { "time_zone": "", "time_format": "" @@ -224,7 +213,6 @@ "previous_type": "string" }, { - "value": "2023-11-08 14:21:16", "option": { "time_zone": "", "time_format": "" @@ -242,7 +230,23 @@ "previous_type": "string" }, { - "value": "上传备份系统成功", + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": false, + "is_delete": false, + "alias_name": "", + "field_name": "kvstoreidx", + "field_type": "int", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "int" + }, + { "option": { "time_zone": "", "time_format": "" @@ -260,7 +264,6 @@ "previous_type": "string" }, { - "value": "redis_binlogbackup", "option": { "time_zone": "", "time_format": "" @@ -278,7 +281,6 @@ "previous_type": "string" }, { - "value": "slave", "option": { "time_zone": "", "time_format": "" @@ -296,7 +298,6 @@ "previous_type": "string" }, { - "value": "127.0.0.1", "option": { "time_zone": "", "time_format": "" @@ -314,7 +315,6 @@ "previous_type": "string" }, { - "value": 30000, "option": { "time_zone": "", "time_format": "" @@ -332,10 +332,26 @@ "previous_type": "int" }, { - "value": "2023-11-08 14:00:16", + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "shard_value", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" + }, + { "option": { "time_zone": 8, - "time_format": "yyyy-MM-dd HH:mm:ss" + "time_format": "strict_date_time_no_millis" }, "is_time": true, "verdict": true, @@ -350,7 +366,6 @@ "previous_type": "string" }, { - "value": "to_backup_system_success", "option": { "time_zone": "", "time_format": "" @@ -366,6 +381,23 @@ "is_built_in": false, "is_dimension": false, "previous_type": "string" + }, + { + "option": { + "time_zone": "", + "time_format": "" + }, + "is_time": false, + "verdict": true, + "is_delete": false, + "alias_name": "", + "field_name": "time_zone", + "field_type": "string", + "description": "", + "is_analyzed": false, + "is_built_in": false, + "is_dimension": false, + "previous_type": "string" } ] } \ No newline at end of file diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json index a93131d684..e05a9e2b3e 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json @@ -8,7 +8,6 @@ "category_id": "host_process", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_fullbackup_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_fullbackup_result.json index 41f63e71b4..8a96abd52c 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/redis_fullbackup_result.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/redis_fullbackup_result.json @@ -8,7 +8,6 @@ "category_id": "host_process", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_keymod.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_keymod.json index f981aa7722..71bad0c89f 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/redis_keymod.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/redis_keymod.json @@ -8,7 +8,6 @@ "category_id": "applications", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "HOST", "target_node_type": "TOPO", diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_slowlog.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_slowlog.json index e33747cba2..84699a2572 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/redis_slowlog.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/redis_slowlog.json @@ -6,13 +6,14 @@ "category_id": "applications", "data_encoding": "UTF-8", "retention": 7, - "allocation_min_days": 3, "es_shards": 3, "target_object_type": "SERVICE", "target_node_type": "TOPO", "target_nodes": [], "params": { - "redis_hosts": ["{{ cmdb_instance.service.labels[\"instance_host\"] }}:{{ cmdb_instance.service.labels[\"instance_port\"] }}"], + "redis_hosts": [ + "{{ cmdb_instance.service.labels[\"instance_host\"] }}:{{ cmdb_instance.service.labels[\"instance_port\"] }}" + ], "redis_password_file": "/home/mysql/.exporter/{{ cmdb_instance.service.labels[\"instance_port\"] }}.conf", "conditions": { "separator": "", @@ -22,20 +23,28 @@ "match_content": "" }, "extra_template_labels": [ - {"key": "$for", "value": "cmdb_instance.scope"}, - {"key": "$body", "value": { - "app": "{{ cmdb_instance.service.labels[\"app\"] }}", - "app_id": "{{ cmdb_instance.service.labels[\"app_id\"] }}", - "bk_cloud_id": "{{ cmdb_instance.service.labels[\"bk_cloud_id\"] }}", - "cluster_name": "{{ cmdb_instance.service.labels[\"cluster_name\"] }}", - "cluster_type": "{{ cmdb_instance.service.labels[\"cluster_type\"] }}", - "instance_host": "{{ cmdb_instance.service.labels[\"instance_host\"] }}", - "instance_port": "{{ cmdb_instance.service.labels[\"instance_port\"] }}", - "instance_role": "{{ cmdb_instance.service.labels[\"instance_role\"] }}", - "cluster_domain": "{{ cmdb_instance.service.labels[\"cluster_domain\"] }}" + { + "key": "$for", + "value": "cmdb_instance.scope" + }, + { + "key": "$body", + "value": { + "app": "{{ cmdb_instance.service.labels[\"app\"] }}", + "app_id": "{{ cmdb_instance.service.labels[\"app_id\"] }}", + "bk_cloud_id": "{{ cmdb_instance.service.labels[\"bk_cloud_id\"] }}", + "cluster_name": "{{ cmdb_instance.service.labels[\"cluster_name\"] }}", + "cluster_type": "{{ cmdb_instance.service.labels[\"cluster_type\"] }}", + "instance_host": "{{ cmdb_instance.service.labels[\"instance_host\"] }}", + "instance_port": "{{ cmdb_instance.service.labels[\"instance_port\"] }}", + "instance_role": "{{ cmdb_instance.service.labels[\"instance_role\"] }}", + "cluster_domain": "{{ cmdb_instance.service.labels[\"cluster_domain\"] }}" } }, - {"key": "$item", "value": "scope"} + { + "key": "$item", + "value": "scope" + } ] } } diff --git a/dbm-ui/backend/dbm_init/json_files/format.py b/dbm-ui/backend/dbm_init/json_files/format.py index 9a82038678..9701f37e6b 100644 --- a/dbm-ui/backend/dbm_init/json_files/format.py +++ b/dbm-ui/backend/dbm_init/json_files/format.py @@ -48,8 +48,6 @@ def format(cls, params: Dict, get_context_method_name: str) -> Dict: """ context = getattr(cls, get_context_method_name)() params.update(context) - if env.BKLOG_STORAGE_CLUSTER_ID: - params["storage_cluster_id"] = env.BKLOG_STORAGE_CLUSTER_ID return params @classmethod diff --git a/dbm-ui/backend/dbm_init/services.py b/dbm-ui/backend/dbm_init/services.py index 025e6d62ac..aa13b8c3ab 100644 --- a/dbm-ui/backend/dbm_init/services.py +++ b/dbm-ui/backend/dbm_init/services.py @@ -139,9 +139,14 @@ def auto_create_bklog_service() -> bool: if hasattr(JsonConfigFormat, f"custom_modify_{log_name}"): bklog_params = JsonConfigFormat.custom_modify(bklog_params, f"custom_modify_{log_name}") # 如果存在对应的环境变量设置了日志自定义的保留天数，则进行更新 - if getattr(env, f"BKLOG_{log_name.upper()}_RETENTION", 0): - retention = getattr(env, f"BKLOG_{log_name.upper()}_RETENTION") - bklog_params.update({"retention": retention, "allocation_min_days": retention // 2}) + retention = getattr(env, f"BKLOG_{log_name.upper()}_RETENTION", env.BKLOG_DEFAULT_RETENTION) + bklog_params["retention"] = retention + # 自定义了 ES 存储集群，则指定 storage_cluster_id + if env.BKLOG_STORAGE_CLUSTER_ID: + bklog_params["storage_cluster_id"] = env.BKLOG_STORAGE_CLUSTER_ID + # 如果集群支持冷热数据，则补充 allocation_min_days，为 retention 的一半即可 + if env.BKLOG_CLUSTER_SUPPORT_HOT_COLD: + bklog_params["allocation_min_days"] = retention // 2 # 获取当前采集项的列表 data = BKLogApi.list_collectors( diff --git a/dbm-ui/backend/env/bklog.py b/dbm-ui/backend/env/bklog.py index 4a1fb2ece9..b38f0e005d 100644 --- a/dbm-ui/backend/env/bklog.py +++ b/dbm-ui/backend/env/bklog.py @@ -15,6 +15,12 @@ # 日志平台存储集群 BKLOG_STORAGE_CLUSTER_ID = get_type_env(key="BKLOG_STORAGE_CLUSTER_ID", _type=int) +# 存储集群是否支持冷热数据功能 +BKLOG_CLUSTER_SUPPORT_HOT_COLD = get_type_env(key="BKLOG_CLUSTER_SUPPORT_HOT_COLD", _type=bool) + +# 日志默认存储天数 +BKLOG_DEFAULT_RETENTION = get_type_env(key="BKLOG_CLUSTER_SUPPORT_HOT_COLD", _type=int, default=7) + # 自定义日志保留天数 BKLOG_MYSQL_BACKUP_RESULT_RETENTION = get_type_env(key="BKLOG_MYSQL_BACKUP_RESULT_RETENTION", _type=int) BKLOG_MYSQL_DBBACKUP_RESULT_RETENTION = get_type_env(key="BKLOG_MYSQL_DBBACKUP_RESULT_RETENTION", _type=int)