From 87423dcf5ce6368bf0d4271583638c0621b080b1 Mon Sep 17 00:00:00 2001 From: seanlook Date: Tue, 3 Dec 2024 15:53:04 +0800 Subject: [PATCH] =?UTF-8?q?feat(mysql):=20dbbackup=20=E5=A2=9E=E5=8A=A0loc?= =?UTF-8?q?k-timeout=20#8380?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../000019_tendbcluster_data.up.sql | 6 +- .../migrations/000020_tendbha_data.up.sql | 6 +- .../migrations/000021_tendbsingle_data.up.sql | 4 +- .../migrations/000041_doris_data.up.sql | 2 +- .../common/go-pubpkg/cmutil/osinfo.go | 16 ++++ .../pkg/components/mysql/install_mysql.go | 14 ++++ .../components/mysql/restore/dbbackup_load.go | 1 + .../dbbackup_loader/xtrabackup_recover.go | 13 +++- .../xtrabackup_recover_repaire.go | 21 +++-- .../mysql/db-tools/mysql-dbbackup/docs/faq.md | 4 +- .../db-tools/mysql-dbbackup/docs/readme.md | 29 ++++++- .../mysql-dbbackup/pkg/config/defaults.go | 3 + .../mysql-dbbackup/pkg/config/public.go | 13 ++++ .../pkg/src/backupexe/dumper.go | 4 +- .../pkg/src/backupexe/dumper_logical.go | 26 ++++++- .../src/backupexe/dumper_logical_mysqldump.go | 2 +- .../pkg/src/backupexe/dumper_physical.go | 78 ++++++++++++------- .../pkg/src/backupexe/loader_physical.go | 16 ++-- .../pkg/src/backupexe/prepareinfo.go | 22 ++---- .../pkg/src/precheck/check_backup_type.go | 3 +- .../db-tools/mysql-dbbackup/pkg/util/misc.go | 49 ++++++++---- .../mysql/db-tools/mysql-monitor/go.mod | 1 + .../mysql/db-tools/mysql-monitor/go.sum | 2 + .../itemscollect/ibdstatistic/report_log.go | 76 +++++++----------- 24 files changed, 265 insertions(+), 146 deletions(-) diff --git a/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql b/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql index 7911bc7581..af6fb154e2 100644 --- a/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql @@ -108,7 +108,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16347,'tendbcluster','backup','dbbackup.ini','LogicalBackup.DefaultsFile','STRING','','','STRING',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 09:50:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16358,'tendbcluster','backup','dbbackup.ini','LogicalBackup.DisableCompress','STRING','false','false | true','BOOL',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-24 21:45:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23728,'tendbcluster','backup','dbbackup.ini','LogicalBackup.ExcludeDatabases','STRING','mysql,sys,test,information_schema,performance_schema,db_infobase','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:40:11',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16346,'tendbcluster','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','--skip-definer','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-25 09:59:19',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16346,'tendbcluster','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14164,'tendbcluster','backup','dbbackup.ini','LogicalBackup.FlushRetryCount','INT','3','','INT',1,0,0,0,0,NULL,'','',-1,NULL,'','2023-03-09 17:36:33','2023-03-22 12:23:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14168,'tendbcluster','backup','dbbackup.ini','LogicalBackup.Regex','STRING','{{.LogicalBackup.Regex}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-03-09 17:36:33','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25532,'tendbcluster','backup','dbbackup.ini','LogicalBackup.Tables','STRING','*','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:42:11',0); @@ -118,7 +118,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23726,'tendbcluster','backup','dbbackup.ini','LogicalBackupMysqldump.ExtraOpt','STRING','','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-11-11 10:22:39',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16343,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.DefaultsFile','STRING','{{.PhysicalBackup.DefaultsFile}}','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 10:24:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23705,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.DisableSlaveMultiThread','STRING','true','true | false','BOOL',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:52',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16354,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','--safe-slave-backup-timeout=60','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-24 21:46:16',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16354,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23716,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.MaxMyisamTables','INT','10','[0, 9999999]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:52',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16363,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.Threads','INT','2','[0, 8]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-10 15:21:23',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16348,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.Throttle','INT','200','[0, 500]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-04-23 12:28:23',0); @@ -327,6 +327,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14361,'tendbcluster','dbconf','MySQL-5.6','mysqld.disconnect_on_expired_password','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'This variable controls how the server handles clients with expired passwords','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14362,'tendbcluster','dbconf','MySQL-5.6','mysqld.div_precision_increment','INT','4','[0,30]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the number of digits by which to increase the scale of the result of division operations performed with the operator','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14363,'tendbcluster','dbconf','MySQL-5.6','mysqld.end_markers_in_json','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Whether optimizer JSON output should add end markers. ','2023-03-09 17:36:33','2023-03-09 17:36:33',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25853,'tendbcluster','dbconf','MySQL-5.6','mysqld.enforce-gtid-consistency','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Depending on the value of this variable, the server enforces GTID consistency by allowing execution of only statements that can be safely logged using a GTID','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14364,'tendbcluster','dbconf','MySQL-5.6','mysqld.eq_range_index_dive_limit','INT','200','[1,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The optimizer will use existing index statistics instead of doing index dives for equality ranges if the number of equality ranges for the index is larger than or equal to this number. If set to 0, index dives are always used.','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14365,'tendbcluster','dbconf','MySQL-5.6','mysqld.event_scheduler','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the status of the Event Scheduler','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14366,'tendbcluster','dbconf','MySQL-5.6','mysqld.expire_logs_days','INT','60','[0, 99]','RANGE',1,0,0,0,1,NULL,NULL,'',0,'',NULL,'2023-03-09 17:36:33','2023-04-26 20:27:21',0); @@ -336,6 +337,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14370,'tendbcluster','dbconf','MySQL-5.6','mysqld.ft_min_word_len','INT','4','[1,84]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'The minimum length of the word to be included in a FULLTEXT index. Note: FULLTEXT indexes must be rebuilt after changing this variable.','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14371,'tendbcluster','dbconf','MySQL-5.6','mysqld.ft_query_expansion_limit','INT','20','[0,1000]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'Number of best matches to use for query expansion','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14372,'tendbcluster','dbconf','MySQL-5.6','mysqld.group_concat_max_len','INT','1024','[4,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The maximum permitted result length in bytes for the GROUP_CONCAT() function','2023-03-09 17:36:33','2023-03-09 17:36:33',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25851,'tendbcluster','dbconf','MySQL-5.6','mysqld.gtid_mode','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Controls whether GTID based logging is enabled and what type of transactions the logs can contain','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14373,'tendbcluster','dbconf','MySQL-5.6','mysqld.host_cache_size','INT','643','[0,65535]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The size of the internal host cache','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14375,'tendbcluster','dbconf','MySQL-5.6','mysqld.innodb_adaptive_flushing','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Specifies whether to dynamically adjust the rate of flushing dirty pages in the InnoDB buffer pool based on the workload.','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14376,'tendbcluster','dbconf','MySQL-5.6','mysqld.innodb_adaptive_flushing_lwm','INT','10','[0,70]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Defines the low water mark representing percentage of redo log capacity at which adaptive flushing is enabled','2023-03-09 17:36:33','2023-03-09 17:36:33',0); diff --git a/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql b/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql index e538a16b0b..e2edc8d296 100644 --- a/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql @@ -106,7 +106,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13010,'tendbha','backup','dbbackup.ini','LogicalBackup.DefaultsFile','STRING','','','STRING',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 09:50:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16311,'tendbha','backup','dbbackup.ini','LogicalBackup.DisableCompress','STRING','false','false | true','BOOL',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-24 21:45:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23729,'tendbha','backup','dbbackup.ini','LogicalBackup.ExcludeDatabases','STRING','mysql,sys,test,information_schema,performance_schema,db_infobase','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:40:11',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16312,'tendbha','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','--skip-definer','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-25 09:59:19',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16312,'tendbha','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13009,'tendbha','backup','dbbackup.ini','LogicalBackup.FlushRetryCount','INT','3','','INT',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-03-22 12:23:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13007,'tendbha','backup','dbbackup.ini','LogicalBackup.Regex','STRING','{{.LogicalBackup.Regex}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25533,'tendbha','backup','dbbackup.ini','LogicalBackup.Tables','STRING','*','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:42:11',0); @@ -116,7 +116,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23724,'tendbha','backup','dbbackup.ini','LogicalBackupMysqldump.ExtraOpt','STRING','','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-11-11 10:22:39',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16293,'tendbha','backup','dbbackup.ini','PhysicalBackup.DefaultsFile','STRING','{{.PhysicalBackup.DefaultsFile}}','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 10:24:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23704,'tendbha','backup','dbbackup.ini','PhysicalBackup.DisableSlaveMultiThread','STRING','true','true | false','BOOL',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:58',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16294,'tendbha','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','--safe-slave-backup-timeout=60','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-24 21:46:16',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16294,'tendbha','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23717,'tendbha','backup','dbbackup.ini','PhysicalBackup.MaxMyisamTables','INT','10','[0, 9999999]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-15 21:03:46',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16290,'tendbha','backup','dbbackup.ini','PhysicalBackup.Threads','INT','2','[0, 8]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-10 15:21:23',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16292,'tendbha','backup','dbbackup.ini','PhysicalBackup.Throttle','INT','200','[0, 500]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-04-23 12:28:18',0); @@ -325,6 +325,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5201,'tendbha','dbconf','MySQL-5.6','mysqld.disconnect_on_expired_password','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'This variable controls how the server handles clients with expired passwords','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5202,'tendbha','dbconf','MySQL-5.6','mysqld.div_precision_increment','INT','4','[0,30]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the number of digits by which to increase the scale of the result of division operations performed with the operator','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5203,'tendbha','dbconf','MySQL-5.6','mysqld.end_markers_in_json','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Whether optimizer JSON output should add end markers. ','2022-06-16 21:39:26','2022-10-20 12:26:09',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25854,'tendbha','dbconf','MySQL-5.6','mysqld.enforce-gtid-consistency','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Depending on the value of this variable, the server enforces GTID consistency by allowing execution of only statements that can be safely logged using a GTID','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5204,'tendbha','dbconf','MySQL-5.6','mysqld.eq_range_index_dive_limit','INT','200','[1,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The optimizer will use existing index statistics instead of doing index dives for equality ranges if the number of equality ranges for the index is larger than or equal to this number. If set to 0, index dives are always used.','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5205,'tendbha','dbconf','MySQL-5.6','mysqld.event_scheduler','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the status of the Event Scheduler','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (85,'tendbha','dbconf','MySQL-5.6','mysqld.expire_logs_days','INT','60','[0, 99]','RANGE',1,0,0,0,1,NULL,NULL,'',0,'',NULL,'2022-04-25 10:00:47','2023-04-26 20:27:21',0); @@ -334,6 +335,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5209,'tendbha','dbconf','MySQL-5.6','mysqld.ft_min_word_len','INT','4','[1,84]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'The minimum length of the word to be included in a FULLTEXT index. Note: FULLTEXT indexes must be rebuilt after changing this variable.','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5210,'tendbha','dbconf','MySQL-5.6','mysqld.ft_query_expansion_limit','INT','20','[0,1000]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'Number of best matches to use for query expansion','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5211,'tendbha','dbconf','MySQL-5.6','mysqld.group_concat_max_len','INT','1024','[4,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The maximum permitted result length in bytes for the GROUP_CONCAT() function','2022-06-16 21:39:26','2022-10-20 12:26:09',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25852,'tendbha','dbconf','MySQL-5.6','mysqld.gtid_mode','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Controls whether GTID based logging is enabled and what type of transactions the logs can contain','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5212,'tendbha','dbconf','MySQL-5.6','mysqld.host_cache_size','INT','643','[0,65535]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The size of the internal host cache','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (86,'tendbha','dbconf','MySQL-5.6','mysqld.init_connect','STRING','','','',2,0,0,0,0,NULL,NULL,NULL,-1,NULL,'','2022-04-25 10:00:47','2024-01-18 17:36:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5213,'tendbha','dbconf','MySQL-5.6','mysqld.innodb_adaptive_flushing','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Specifies whether to dynamically adjust the rate of flushing dirty pages in the InnoDB buffer pool based on the workload.','2022-06-16 21:39:26','2022-10-20 12:26:09',0); diff --git a/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql b/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql index b02cf4f656..beefdcb56f 100644 --- a/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql @@ -99,7 +99,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16389,'tendbsingle','backup','dbbackup.ini','LogicalBackup.DefaultsFile','STRING','','','STRING',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 09:50:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16400,'tendbsingle','backup','dbbackup.ini','LogicalBackup.DisableCompress','STRING','false','false | true','BOOL',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-24 21:45:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23734,'tendbsingle','backup','dbbackup.ini','LogicalBackup.ExcludeDatabases','STRING','mysql,sys,test,information_schema,performance_schema,db_infobase','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:40:11',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16388,'tendbsingle','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','--skip-definer','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-25 09:59:19',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16388,'tendbsingle','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13051,'tendbsingle','backup','dbbackup.ini','LogicalBackup.FlushRetryCount','INT','3','','INT',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-03-22 12:23:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13055,'tendbsingle','backup','dbbackup.ini','LogicalBackup.Regex','STRING','{{.LogicalBackup.Regex}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25534,'tendbsingle','backup','dbbackup.ini','LogicalBackup.Tables','STRING','*','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:42:11',0); @@ -109,7 +109,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23732,'tendbsingle','backup','dbbackup.ini','LogicalBackupMysqldump.ExtraOpt','STRING','','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-11-11 10:22:39',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16385,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.DefaultsFile','STRING','{{.PhysicalBackup.DefaultsFile}}','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 10:24:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23702,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.DisableSlaveMultiThread','STRING','true','true | false','BOOL',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:36:04',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16396,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','--safe-slave-backup-timeout=60','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-24 21:46:16',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16396,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23718,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.MaxMyisamTables','INT','10','[0, 9999999]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:52',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16405,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.Threads','INT','2','[0, 8]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-10 15:21:23',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16390,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.Throttle','INT','200','[0, 500]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-04-23 12:28:34',0); diff --git a/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql b/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql index 0d0f28bf4a..fb8445e812 100644 --- a/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql @@ -1050,7 +1050,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24449,'doris','dbconf','2.1.5','fe.tablet_rebalancer_type','STRING','BeLoad','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','tablet_rebalancer_type','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24450,'doris','dbconf','2.1.5','fe.tablet_repair_delay_factor_second','STRING','60','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','tablet_repair_delay_factor_second','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24451,'doris','dbconf','2.1.5','fe.tablet_stat_update_interval_second','STRING','300','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','tablet_stat_update_interval_second','2023-01-13 11:26:00','2023-04-17 17:10:41',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24452,'doris','dbconf','2.1.5','fe.table_name_length_limit','STRING','64','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','table_name_length_limit','2023-01-13 11:26:00','2023-04-17 17:10:41',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24452,'doris','dbconf','2.1.5','fe.table_name_length_limit','STRING','128','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','table_name_length_limit','2023-01-13 11:26:00','2024-12-02 15:12:20',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24453,'doris','dbconf','2.1.5','fe.thrift_backlog_num','STRING','1024','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','thrift_backlog_num','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24454,'doris','dbconf','2.1.5','fe.thrift_client_timeout_ms','STRING','0','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','thrift_client_timeout_ms','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24455,'doris','dbconf','2.1.5','fe.thrift_server_max_worker_threads','STRING','4096','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','thrift_server_max_worker_threads','2023-01-13 11:26:00','2023-04-17 17:10:41',0); diff --git a/dbm-services/common/go-pubpkg/cmutil/osinfo.go b/dbm-services/common/go-pubpkg/cmutil/osinfo.go index 6469381685..56d497cca7 100644 --- a/dbm-services/common/go-pubpkg/cmutil/osinfo.go +++ b/dbm-services/common/go-pubpkg/cmutil/osinfo.go @@ -2,6 +2,7 @@ package cmutil import ( "path/filepath" + "regexp" "strings" "github.com/pkg/errors" @@ -155,3 +156,18 @@ func IsSameTopLevelDir(dir1, dir2 string) (bool, error) { } return topDir1 == topDir2, nil } + +// GetGlibcVersion get os glibc version using ExeCommand("ldd --version |grep libc") +func GetGlibcVersion() (string, error) { + outStr, errStr, err := ExecCommand(false, "", + "/usr/bin/ldd", "--version", "|", "grep", "libc") + if err != nil { + return "", errors.WithMessage(err, errStr) + } + verMatch := regexp.MustCompile(`ldd \(.*\) (\d+\.\d+)`) + ms := verMatch.FindStringSubmatch(outStr) + if len(ms) == 2 { + return ms[1], nil + } + return "", errors.New("ldd --version | grep glibc fail to get glibc version") +} diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go index 67ed737ae7..d5eb93d64e 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go @@ -264,6 +264,7 @@ func (i *InstallMySQLComp) InitDefaultParam() (err error) { i.Checkfunc = append(i.Checkfunc, i.precheckMysqlDir) i.Checkfunc = append(i.Checkfunc, i.precheckMysqlProcess) i.Checkfunc = append(i.Checkfunc, i.precheckMysqlPackageBitOS) + i.Checkfunc = append(i.Checkfunc, i.precheckGlibcVersion) i.Checkfunc = append(i.Checkfunc, i.Params.Medium.Check) i.Params.PartitionYWAccount.AccessHosts = []string{ i.Params.Host, @@ -361,6 +362,19 @@ func (i *InstallMySQLComp) precheckMysqlPackageBitOS() error { return nil } +func (i *InstallMySQLComp) precheckGlibcVersion() error { + glibcVer, err := cmutil.GetGlibcVersion() + if err != nil { + logger.Error("failed to glibc version, err:%s", err.Error()) + return err + } + // mysql 8.0 不安装在 tlinux1.2上 (mysqld能运行，但周边 xtrabackup 8.0 依赖 glibc>=2.14) + if cmutil.MySQLVersionParse(i.Params.MysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") && glibcVer < "2.14" { + return errors.Errorf("glibc version %s it not allowed to install %s", glibcVer, i.Params.MysqlVersion) + } + return nil +} + // initMySQLInstanceMem TODO // GetInstMemByIP 返回的内存单位是 MB func (i *InstallMySQLComp) initMySQLInstanceMem() (err error) { diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go index 15c36571ee..fbd27fc14c 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go @@ -133,6 +133,7 @@ func (m *DBLoader) chooseDBBackupLoader() error { QpressTool: m.Tools.MustGet(tools.ToolQPress), LoaderDir: m.targetDir, StorageType: strings.ToLower(m.indexObj.StorageEngine), + MySQLVersion: m.BackupInfo.indexObj.MysqlVersion, }, } } else { diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go index a3a5703c7a..de185c2852 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go @@ -30,7 +30,8 @@ type Xtrabackup struct { // 在 PreRun 时初始化，本地实例的配置文件 myCnf *util.CnfFile - StorageType string `json:"-"` + StorageType string `json:"-"` + MySQLVersion string `json:"-"` } // PreRun 以下所有步骤必须可重试 @@ -79,7 +80,7 @@ func (x *Xtrabackup) PostRun() (err error) { return err } - logger.Info("start local mysqld with skip-grant-tables") + logger.Info("start local mysqld with --skip-grant-tables --skip-slave-start") // 启动mysql-修复权限 startParam := computil.StartMySQLParam{ MediaDir: cst.MysqldInstallPath, @@ -88,6 +89,7 @@ func (x *Xtrabackup) PostRun() (err error) { MySQLPwd: x.TgtInstance.Pwd, Socket: x.TgtInstance.Socket, SkipGrantTables: true, // 以 skip-grant-tables 启动来修复 ADMIN + SkipSlaveFlag: true, } if _, err = startParam.StartMysqlInstance(); err != nil { return errors.WithMessage(err, "start mysqld after xtrabackup") @@ -106,6 +108,11 @@ func (x *Xtrabackup) PostRun() (err error) { serverVersion = "5.7.20" // fake } } + if _, err = x.dbWorker.Exec("reset slave all"); err != nil { + logger.Warn("failed to reset slave all, err", err.Error()) + } + + //return nil logger.Info("repair user 'ADMIN' host and password") // 物理备份，ADMIN密码与 backup instance(cluster?) 相同，修复成 // 修复ADMIN用户，而不是 x.TgtInstance.User，主要是修复 host，密码修复成临时用户 DBM_JOB_xxx 的密码 @@ -130,10 +137,12 @@ func (x *Xtrabackup) PostRun() (err error) { logger.Info("restart local mysqld %d", x.TgtInstance.Port) // 重启mysql（去掉 skip-grant-tables） startParam.SkipGrantTables = false + startParam.SkipSlaveFlag = false startParam.MySQLUser = native.DBUserAdmin if _, err := startParam.RestartMysqlInstance(); err != nil { return errors.WithMessage(err, "RestartMysqlInstance") } + logger.Info("reconnect use ADMIN and temp_job_user pwd(already repaired) %d", x.TgtInstance.Port) tmpAdminPassInst := deepcopy.Copy(x.TgtInstance).(native.InsObject) tmpAdminPassInst.User = native.DBUserAdmin diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go index 33e4fd013b..91502307fb 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go @@ -185,8 +185,8 @@ func (x *Xtrabackup) RepairPrivileges() error { var batchSQLs []string // delete src host's ADMIN/sync user - sql1 := fmt.Sprintf("DELETE FROM mysql.user WHERE `user` IN (%s) AND `host` = %s;", myUsersUnsafe, srcHostUnsafe) - batchSQLs = append(batchSQLs, sql1) + batchSQLs = append(batchSQLs, + fmt.Sprintf("DELETE FROM mysql.user WHERE `user` IN (%s) AND `host` = %s;", myUsersUnsafe, srcHostUnsafe)) // update src host to new, but not ADMIN/sync/repl sql2s := []string{ @@ -203,16 +203,15 @@ func (x *Xtrabackup) RepairPrivileges() error { tgtHostUnsafe, srcHostUnsafe, myUsersUnsafe, ), } - batchSQLs = append(batchSQLs, sql2s...) - - /* - // delete src host users, but not localhost - sql3 := fmt.Sprintf( - "DELETE FROM mysql.user WHERE `host` IN(%s);", srcHostUnsafe, - ) - batchSQLs = append(batchSQLs, sql3) - */ + if cmutil.MySQLVersionParse(x.MySQLVersion) >= cmutil.MySQLVersionParse("8.0.0") { + // mysql 8.0 的 BACKUP_ADMIN 权限记录在 mysql.global_grants 中 + sql2s = append(sql2s, fmt.Sprintf( + "UPDATE mysql.global_grants SET `HOST`=%s WHERE `HOST`=%s AND User NOT IN (%s);", + tgtHostUnsafe, srcHostUnsafe, myUsersUnsafe, + )) + } + batchSQLs = append(batchSQLs, sql2s...) // flush sql4 := fmt.Sprintf("flush privileges;") batchSQLs = append(batchSQLs, sql4) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md index 7a52e750a1..2bcb03b016 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md @@ -77,4 +77,6 @@ IOLimitMBPerSec = 300 ### 常见备份失败处理 1. log copying being too slow -it looks like InnoDB log has wrapped around before xtrabackup could process all records due to either log copying being too slow, or log files being too small. \ No newline at end of file +it looks like InnoDB log has wrapped around before xtrabackup could process all records due to either log copying being too slow, or log files being too small. + 实例写入速度比备份速度快，可能是刚刚备份时间段有大批量 DML 操作，修改备份开始时间，或者加快备份速度: + - 调大 PhysicalBackup.Throttle, 调大 PhysicalBackup.Threads \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md index e4f78593bd..0730280752 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md @@ -319,4 +319,31 @@ Threads = 4 CopyBack = false IndexFilePath = xx DefaultsFile = /etc/my.cnf.3306 -``` \ No newline at end of file +``` + +## 参数解释 +### Public +- Public.KillLongQueryTime + 发起备份前检查长 sql，如果超过这个时间则马上自动 kill。需要备份账号有 super 权限 + 默认为 0 则不 kill，对 mydumper / xtrabackup 有效 + +- Public.FtwrlWaitTimeout + 发起备份前检查长 sql，(如果不自动 kill/ kill失败) 则等待长 sql 多久后，放弃 ftwrl，放弃备份。 + 此时还未发起 `FLUSH TABLE WITH READ LOCK` 命令 + 默认 120s，对 mydumper / xtrabackup 有效 + +- Public.AcquireLockWaitTimeout + 备份加锁超时，比如 `LOCK TABLES FOR BACKUP` / `FLUSH TABLE WITH READ LOCK`，相当于 `set session lock-wait-timeout=xxx` + 默认 10s 超时，对 mydumper / xtrabackup 有效 + +- Public.BackupTimeOut + 备份超时结束时间，只对 master 有效，用于保护 master 不被备份影响。 + 默认`09:00:00`，即备份执行到这个时间点还未结束，则退出 + +- Public.OldFileLeftDay + 备份文件本地最大保留时间天数，每次备份之前会先清理旧的备份。如果备份空间不足，可能会继续清理备份文件，优先保证备份成功 + +- Public.IOLimitMasterFactor + master机器专用限速因子，因为备份速度可能有多个选项来控制 + - master 文件io打包限速: `Public.IOLimitMBPerSec * IOLimitMasterFactor` + - 物理备份限速: `PhysicalBackup.Throttle * IOLimitMasterFactor` \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go index 5c1f265e88..73ea9ac8f6 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go @@ -25,6 +25,9 @@ func SetDefaults() { viper.SetDefault("Public.BackupDir", "/data/dbbak") //viper.SetDefault("Public.MysqlCharset", "binary") viper.SetDefault("Public.BackupTimeOut", "09:00:01") + viper.SetDefault("Public.KillLongQueryTime", 0) + viper.SetDefault("Public.FtwrlWaitTimeout", 120) + viper.SetDefault("Public.AcquireLockWaitTimeout", 10) viper.SetDefault("PhysicalBackup.MaxMyisamTables", 10) viper.SetDefault("PhysicalBackup.DisableSlaveMultiThread", false) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go index cdf209a8f0..47732a561c 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go @@ -75,6 +75,19 @@ type Public struct { // EncryptOpt backup files encrypt options EncryptOpt *cmutil.EncryptOpt `ini:"EncryptOpt"` + // KillLongQueryTime 如果有长 sql，则等待长 sql 多久后，kill掉长 sql. if 0 not kill. Default 0 + // mydumper --kill-long-queries --long-query-guard xx + // xtrabackup --kill-long-queries-timeout + KillLongQueryTime int `ini:"KillLongQueryTime"` + // FtwrlWaitTimeout 如果有长 sql，则等待长 sql 多久后，放弃 ftwrl，放弃备份(FTWRL 还没运行). Default 120s + // xtrabackup --ftwrl-wait-timeout + // mydumper --long-query-guard + FtwrlWaitTimeout int `ini:"FtwrlWaitTimeout"` + // AcquireLockWaitTimeout If LOCK TABLES FOR BACKUP does not return within given timeout, abort the backup. + // issue lock to mysqld: default 10s + // lock-ddl-timeout(xtrabackup57), backup-lock-timeout(xtrabackup80) --lock-wait-timeout(mydumper) + AcquireLockWaitTimeout int `ini:"AcquireLockWaitTimeout"` + cnfFilename string targetName string // isFullBackup 1: true, -1: false, 0: unknown diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go index afe95d0fe4..1980fdd218 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/validate" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" @@ -15,7 +16,6 @@ import ( "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck" - "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util" ) // Dumper TODO @@ -45,7 +45,7 @@ func BuildDumper(cnf *config.BackupConfig, db *sql.DB) (dumper Dumper, err error if strings.ToLower(cnf.Public.BackupType) == cst.BackupLogical { if cnf.LogicalBackup.UseMysqldump == cst.LogicalMysqldumpAuto || cnf.LogicalBackup.UseMysqldump == "" { - if glibcVer, err := util.GetGlibcVersion(); err != nil { + if glibcVer, err := cmutil.GetGlibcVersion(); err != nil { logger.Log.Warn("failed to glibc version, err:", err) } else if glibcVer < "2.14" { // mydumper need glibc version >= 2.14 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go index 4ceebf8e8a..ea23d54bab 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go @@ -65,15 +65,29 @@ func (l *LogicalDumper) Execute(enableTimeOut bool) error { "-u", l.cnf.Public.MysqlUser, "-p", l.cnf.Public.MysqlPasswd, "-o", filepath.Join(l.cnf.Public.BackupDir, l.cnf.Public.TargetName()), + "--trx-consistency-only", + "--long-query-retry-interval=10", fmt.Sprintf("--long-query-retries=%d", l.cnf.LogicalBackup.FlushRetryCount), fmt.Sprintf("--set-names=%s", l.cnf.Public.MysqlCharset), fmt.Sprintf("--chunk-filesize=%d", l.cnf.LogicalBackup.ChunkFilesize), fmt.Sprintf("--threads=%d", l.cnf.LogicalBackup.Threads), - "--trx-consistency-only", - "--long-query-retry-interval=10", // "--disk-limits=1GB:5GB", } - + if l.cnf.Public.KillLongQueryTime > 0 { + args = append(args, "--kill-long-queries", + fmt.Sprintf("--long-query-guard=%d", l.cnf.Public.KillLongQueryTime)) + } else { + if l.cnf.Public.FtwrlWaitTimeout > 0 { + args = append(args, fmt.Sprintf("--long-query-guard=%d", l.cnf.Public.FtwrlWaitTimeout)) + } else { + args = append(args, "--long-query-guard=999999") // 不退出 + } + } + /* + if l.cnf.Public.AcquireLockWaitTimeout > 0 { + args = append(args, fmt.Sprintf("--lock-wait-timeout=%d", l.cnf.Public.AcquireLockWaitTimeout)) + } + */ if !l.cnf.LogicalBackup.DisableCompress { args = append(args, "--compress") } @@ -124,7 +138,11 @@ func (l *LogicalDumper) Execute(enableTimeOut bool) error { }...) } } - // ToDo extropt + /* + if l.cnf.LogicalBackup.ExtraOpt != "" { + args = append(args, l.cnf.LogicalBackup.ExtraOpt) + } + */ var cmd *exec.Cmd if enableTimeOut { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go index d64dbd96ec..67ec171b60 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go @@ -189,7 +189,7 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) { "--single-transaction", "--master-data=2", "--max-allowed-packet=1G", "--no-autocommit", "--hex-blob", - // "--set-gtid-purged=off", // 5.7 需要 + // "--set-gtid-purged=off", // 5.7 需要 如果支持 } args = append(args, "--default-character-set", l.cnf.Public.MysqlCharset) if l.cnf.Public.MysqlRole == cst.RoleSlave { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go index b8fc460ff9..9c901c66c9 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go @@ -1,7 +1,6 @@ package backupexe import ( - "bytes" "context" "fmt" "os" @@ -90,50 +89,62 @@ func (p *PhysicalDumper) buildArgs() []string { fmt.Sprintf("--password=%s", p.cnf.Public.MysqlPasswd), "--compress", } + if p.cnf.PhysicalBackup.Threads > 0 { + args = append(args, []string{ + fmt.Sprintf("--compress-threads=%d", p.cnf.PhysicalBackup.Threads), + fmt.Sprintf("--parallel=%d", p.cnf.PhysicalBackup.Threads), + }...) + } targetPath := filepath.Join(p.cnf.Public.BackupDir, p.cnf.Public.TargetName()) if strings.Compare(p.mysqlVersion, "005007000") < 0 { args = append(args, targetPath) } else { - args = append(args, - fmt.Sprintf("--target-dir=%s", targetPath), - "--backup", "--lock-ddl", - ) + args = append(args, fmt.Sprintf("--target-dir=%s", targetPath), "--backup") } - - if p.cnf.PhysicalBackup.Threads > 0 { - args = append(args, []string{ - fmt.Sprintf("--compress-threads=%d", p.cnf.PhysicalBackup.Threads), - fmt.Sprintf("--parallel=%d", p.cnf.PhysicalBackup.Threads), - }...) + if strings.Compare(p.mysqlVersion, "005007000") > 0 { + args = append(args, "--lock-ddl") + if p.cnf.Public.AcquireLockWaitTimeout > 0 { + args = append(args, fmt.Sprintf("--lock-ddl-timeout=%d", p.cnf.Public.AcquireLockWaitTimeout)) + } + if p.cnf.Public.FtwrlWaitTimeout > 0 { + args = append(args, fmt.Sprintf("--ftwrl-wait-timeout=%d", p.cnf.Public.FtwrlWaitTimeout)) + } + if strings.Compare(p.mysqlVersion, "008000000") < 0 { + args = append(args, "--binlog-info=ON") // ver >=5.7 and ver < 8.0 + } + } + if strings.Compare(p.mysqlVersion, "005006000") > 0 { + if p.cnf.Public.KillLongQueryTime > 0 { + args = append(args, fmt.Sprintf("--kill-long-queries-timeout=%d", p.cnf.Public.KillLongQueryTime)) + } } if p.cnf.PhysicalBackup.Throttle > 0 { - args = append(args, []string{ - fmt.Sprintf("--throttle=%d", p.cnf.PhysicalBackup.Throttle), - }...) + args = append(args, fmt.Sprintf("--throttle=%d", p.cnf.PhysicalBackup.Throttle)) } if strings.ToLower(p.cnf.Public.MysqlRole) == cst.RoleSlave { - args = append(args, []string{ - "--slave-info", - //"--safe-slave-backup", - }...) + // --safe-slave-backup + args = append(args, "--slave-info") } if strings.Compare(p.mysqlVersion, "008000000") >= 0 { if p.isOfficial { args = append(args, "--skip-strict") } + if p.cnf.Public.AcquireLockWaitTimeout > 0 { + args = append(args, fmt.Sprintf("--backup-lock-timeout=%d", p.cnf.Public.AcquireLockWaitTimeout)) + } } else { // xtrabackup_80 has no this args, and will report errors args = append(args, "--no-timestamp", "--lazy-backup-non-innodb", "--wait-last-flush=2") args = append(args, fmt.Sprintf("--ibbackup=%s", filepath.Join(p.dbbackupHome, p.innodbCmd.xtrabackupBin))) - if strings.Compare(p.mysqlVersion, "005007000") > 0 { - args = append(args, "--binlog-info=ON") - } } - - // ToDo extropt + /* + if p.cnf.PhysicalBackup.ExtraOpt != "" { + args = append(args, p.cnf.PhysicalBackup.ExtraOpt) + } + */ return args } @@ -191,8 +202,9 @@ func (p *PhysicalDumper) Execute(enableTimeOut bool) error { fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " "))) } - outFile, err := os.Create(filepath.Join(logger.GetLogDir(), - fmt.Sprintf("xtrabackup_%d_%d.log", p.cnf.Public.MysqlPort, int(time.Now().Weekday())))) + xtrabackupLogFile := filepath.Join(logger.GetLogDir(), + fmt.Sprintf("xtrabackup_%d_%d.log", p.cnf.Public.MysqlPort, int(time.Now().Weekday()))) + outFile, err := os.OpenFile(xtrabackupLogFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { logger.Log.Error("create log file failed: ", err) return err @@ -202,15 +214,21 @@ func (p *PhysicalDumper) Execute(enableTimeOut bool) error { }() cmd.Stdout = outFile - //cmd.Stderr = outFile - var stderr bytes.Buffer - cmd.Stderr = &stderr + cmd.Stderr = outFile // xtrabackup 的运行日志都是打印在 stderr ... unbelievable logger.Log.Info("xtrabackup command: ", cmd.String()) err = cmd.Run() if err != nil { - logger.Log.Error("run physical backup failed: ", err, stderr.String()) - return errors.WithMessage(err, stderr.String()) + errStrPrefix := fmt.Sprintf("tail 10 error from %s", xtrabackupLogFile) + errStrDetail, _ := util.GrepLinesFromFile(xtrabackupLogFile, []string{"ERROR", "fatal"}, 10, false, true) + if len(errStrDetail) > 0 { + logger.Log.Info(errStrPrefix) + logger.Log.Error(errStrDetail) + } else { + logger.Log.Warn("tail can not find more detail error message from ", xtrabackupLogFile) + } + logger.Log.Error("run physical backup failed", err, p.cnf.Public.MysqlPort) + return errors.WithMessagef(err, fmt.Sprintf("%s\n%s", errStrPrefix, errStrDetail)) } return nil } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go index dcea221446..f1f892468e 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go @@ -152,14 +152,13 @@ func (p *PhysicalLoader) apply() error { if err != nil { logger.Log.Error("physical apply failed: ", err, errStr) // 尝试读取 xtrabackup.log 里 ERROR 关键字 - grepError := []string{"grep", "-Ei", "ERROR|fatal", logfile, "|", "tail", "-5"} errStrPrefix := fmt.Sprintf("tail 5 error from %s", logfile) - errStrDetail, _, _ := cmutil.ExecCommand(true, "", grepError[0], grepError[1:]...) - if len(strings.TrimSpace(errStrDetail)) > 0 { + errStrDetail, _ := util.GrepLinesFromFile(logfile, []string{"ERROR", "fatal"}, 5, false, true) + if len(errStrDetail) > 0 { logger.Log.Info(errStrPrefix) logger.Log.Error(errStrDetail) } else { - logger.Log.Warn("can not find more detail error message from ", logfile) + logger.Log.Warn("tail can not find more detail error message from ", logfile) } return errors.WithMessagef(err, fmt.Sprintf("%s: %s\n%s", errStr, errStrPrefix, errStrDetail)) } @@ -217,16 +216,15 @@ func (p *PhysicalLoader) load() error { if err != nil { logger.Log.Error("xtrabackup copy data failed: ", err, errStr) // 尝试读取 xtrabackup.log 里 ERROR 关键字 - grepError := []string{"grep", "-Ei", "ERROR|fatal", logfile, "|", "tail", "-5"} errStrPrefix := fmt.Sprintf("tail 5 error from %s", logfile) - errStrDetail, _, _ := cmutil.ExecCommand(true, "", grepError[0], grepError[1:]...) - if len(strings.TrimSpace(errStrDetail)) > 0 { + errStrDetail, _ := util.GrepLinesFromFile(logfile, []string{"ERROR", "fatal"}, 5, false, true) + if len(errStrDetail) > 0 { logger.Log.Info(errStrPrefix) logger.Log.Error(errStrDetail) } else { - logger.Log.Warn("can not find more detail error message from ", logfile) + logger.Log.Warn("tail can not find more detail error message from ", logfile) } - return errors.WithMessagef(err, fmt.Sprintf("%s: %s\n%s", errStr, errStrPrefix, errStrDetail)) + return errors.WithMessagef(err, fmt.Sprintf("%s\n%s", errStrPrefix, errStrDetail)) } logger.Log.Info("xtrabackup recover success: ", outStr) return nil diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go index 9a4c05b019..b82735b693 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go @@ -43,25 +43,17 @@ func parseMysqldumpMetadata(sqlFilePath string) (*mydumperMetadata, error) { var bufScanner *bufio.Scanner if strings.HasSuffix(sqlFilePath, cst.ZstdSuffix) { - /* - var compressedBuf = make([]byte, 2048) - if _, err := sqlFile.Read(compressedBuf); err != nil && err != io.EOF { - return nil, errors.WithMessagef(err, "read first 10240 bytes from %s", sqlFilePath) - } - var textBuf = make([]byte, 2048) - dec, _ := zstd.NewReader(bytes.NewBuffer(compressedBuf), zstd.WithDecoderConcurrency(1)) - defer dec.Close() - if _, err := dec.Read(textBuf); err != nil { - logger.Log.Warnf("zstd decode first 10240 bytes failed from %s, err:%s", sqlFilePath, err.Error()) - } - bufScanner = bufio.NewScanner(bytes.NewBuffer(textBuf)) - */ - cmds := []string{"cut", "-b", "-2048", sqlFilePath, "|", CmdZstd, "-d", "-c"} + cmds := []string{"cut", "-b", "-4096", sqlFilePath, "|", CmdZstd, "-d", "-c"} outBuf, _, err := cmutil.ExecCommandReturnBytes(true, "", cmds[0], cmds[1:]...) + + if len(outBuf) < 100 { // 返回小于这个长度，肯定非法了，重试一遍 + cmds = []string{"cut", "-b", "-10240", sqlFilePath, "|", CmdZstd, "-d", "-c"} + outBuf, _, err = cmutil.ExecCommandReturnBytes(true, "", cmds[0], cmds[1:]...) + } if err != nil { logger.Log.Warnf("zstd decode first 2048 bytes failed from %s, err:%s", sqlFilePath, err.Error()) } - if len(outBuf) < 100 { // 返回小于这个长度，肯定非法了 + if len(outBuf) < 100 { // 返回小于这个长度，非法报错 return nil, errors.Errorf("failed to get binlog position from zst file %s", sqlFilePath) } bufScanner = bufio.NewScanner(bytes.NewBuffer(outBuf)) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go index 263f2f03e3..5c78783db1 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go @@ -11,6 +11,7 @@ package precheck import ( + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" @@ -37,7 +38,7 @@ func CheckBackupType(cnf *config.BackupConfig, storageEngine string) error { } else { cnf.Public.BackupType = cst.BackupLogical } - if glibcVer, err := util.GetGlibcVersion(); err != nil { + if glibcVer, err := cmutil.GetGlibcVersion(); err != nil { logger.Log.Warn("failed to glibc version, err:", err) } else if glibcVer < "2.14" { // mydumper need glibc version >= 2.14 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go index 75e8808260..1db01b8cba 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go @@ -14,6 +14,7 @@ import ( "syscall" "github.com/pkg/errors" + "github.com/spf13/cast" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" @@ -184,21 +185,6 @@ func CheckDiskSpace(backupDir string, mysqlPort int, backupSize uint64) (sizeLef return sizeLeft, nil } -func GetGlibcVersion() (string, error) { - //ExeCommand("ldd --version |grep libc") - outStr, errStr, err := cmutil.ExecCommand(false, "", - "/usr/bin/ldd", "--version", "|", "grep", "libc") - if err != nil { - return "", errors.WithMessage(err, errStr) - } - verMatch := regexp.MustCompile(`ldd \(.*\) (\d+\.\d+)`) - ms := verMatch.FindStringSubmatch(outStr) - if len(ms) == 2 { - return ms[1], nil - } - return "", errors.New("ldd --version | grep libc fail to get glibc version") -} - // ExeCommand execute shell command func ExeCommand(cmdStr string) error { res, exeErr := exec.Command("/bin/bash", "-c", cmdStr).CombinedOutput() @@ -286,3 +272,36 @@ func FindBackupConfigFiles(dir string) ([]string, error) { } return cnfFilesNew, nil } + +// GrepLinesFromFile 从文件里面 grep 错误关键字 +// 如果不指定 keywords，则直接 tail / head 文件行 +func GrepLinesFromFile(logFilePath string, keywords []string, linesRet int, sensitive bool, tail bool) (string, error) { + var grepCommand []string + lineNum := "-" + cast.ToString(linesRet) + if len(keywords) > 0 { + grepExpr := strings.Join(keywords, "|") + if sensitive { + grepCommand = append(grepCommand, "grep", "-E") + } else { + grepCommand = append(grepCommand, "grep", "-Ei") + } + grepCommand = append(grepCommand, grepExpr, logFilePath) + if tail { + grepCommand = append(grepCommand, "|", "tail", lineNum) + } else { + grepCommand = append(grepCommand, "|", "head", lineNum) + } + } else { + if tail { + grepCommand = append(grepCommand, "tail", lineNum, logFilePath) + } else { + grepCommand = append(grepCommand, "head", lineNum, logFilePath) + } + } + errStrDetail, cmdStdErr, err := cmutil.ExecCommand(true, "", grepCommand[0], grepCommand[1:]...) + if strings.TrimSpace(errStrDetail) != "" { + return errStrDetail, nil + } else { + return "", errors.WithMessage(err, cmdStdErr) + } +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/go.mod b/dbm-services/mysql/db-tools/mysql-monitor/go.mod index 70edb8ab12..7479b73d27 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/go.mod +++ b/dbm-services/mysql/db-tools/mysql-monitor/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-playground/validator/v10 v10.15.4 github.com/go-sql-driver/mysql v1.6.0 github.com/jmoiron/sqlx v1.3.5 + github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b github.com/juju/ratelimit v1.0.2 github.com/olekukonko/tablewriter v0.0.5 github.com/pingcap/errors v0.11.4 diff --git a/dbm-services/mysql/db-tools/mysql-monitor/go.sum b/dbm-services/mysql/db-tools/mysql-monitor/go.sum index 4fc3e5d5c4..eebfdc097c 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/go.sum +++ b/dbm-services/mysql/db-tools/mysql-monitor/go.sum @@ -145,6 +145,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b h1:FQ7+9fxhyp82ks9vAuyPzG0/vVbWwMwLJ+P6yJI5FN8= +github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b/go.mod h1:HMcgvsgd0Fjj4XXDkbjdmlbI505rUPBs6WBMYg2pXks= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go index 580cf09081..75423dce7d 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go @@ -1,34 +1,38 @@ package ibdstatistic import ( - "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" - "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" - "encoding/json" - "fmt" "log/slog" "os" "path/filepath" "slices" "time" + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/reportlog" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" + "github.com/pkg/errors" ) type tableSizeStruct struct { - BkCloudId int `json:"bk_cloud_id"` - BkBizId int `json:"bk_biz_id"` - ImmuteDomain string `json:"cluster_domain"` - DBModule int `json:"db_module"` - MachineType string `json:"machine_type"` - Ip string `json:"instance_host"` - Port int `json:"instance_port"` - Role string `json:"instance_role"` - ServiceInstanceId int64 `json:"bk_target_service_instance_id"` - OriginalDBName string `json:"original_database_name"` - DBName string `json:"database_name"` - DBSize int64 `json:"database_size"` - TableName string `json:"table_name"` - TableSize int64 `json:"table_size"` + // ReportTime time.RFC3339, format like 024-11-29T11:19:02+08:00 + ReportTime time.Time `json:"report_time"` + BkCloudId int `json:"bk_cloud_id"` + BkBizId int `json:"bk_biz_id"` + ImmuteDomain string `json:"cluster_domain"` + DBModule int `json:"db_module"` + MachineType string `json:"machine_type"` + Ip string `json:"instance_host"` + Port int `json:"instance_port"` + Role string `json:"instance_role"` + ServiceInstanceId int64 `json:"bk_target_service_instance_id"` + // OriginalDBName original DBName for spider remote(with shard_id suffix) + OriginalDBName string `json:"original_database_name"` + DBName string `json:"database_name"` + DBSize int64 `json:"database_size"` + TableName string `json:"table_name"` + TableSize int64 `json:"table_size"` } func reportLog(result map[string]map[string]int64) error { @@ -38,26 +42,11 @@ func reportLog(result map[string]map[string]int64) error { slog.Error("failed to create database size reports directory", slog.String("error", err.Error())) return errors.Wrap(err, "failed to create database size reports directory") } - - filePath := filepath.Join( - dbsizeReportBaseDir, - fmt.Sprintf(`report.log.%d`, time.Now().Weekday()), - ) - err = os.RemoveAll(filePath) - if err != nil { - slog.Error("failed to remove database size reports directory", slog.String("error", err.Error())) - return errors.Wrap(err, "failed to remove database size reports directory") - } - - f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + resultReport, err := reportlog.NewReporter(dbsizeReportBaseDir, "report.log", nil) if err != nil { - slog.Error("failed to open log file", "file", filePath) - return errors.Wrap(err, "failed to open file") + return err } - defer func() { - _ = f.Close() - }() - + reportTs := cmutil.TimeToSecondPrecision(time.Now()) for originalDBName, dbInfo := range result { // 根据 dbm 枚举约定, remote 是 tendbcluster 的存储机器类型 dbName := originalDBName @@ -98,18 +87,11 @@ func reportLog(result map[string]map[string]int64) error { for _, row := range tablesInfo { row.DBSize = dbSize - b, err := json.Marshal(row) - if err != nil { - slog.Error("ibd-statistic report", slog.String("error", err.Error())) - return errors.Wrap(err, "failed to marshal row") - } - - b = append(b, '\n') - _, err = f.Write(b) - if err != nil { - slog.Error("ibd-statistic report", slog.String("error", err.Error())) - return errors.Wrap(err, "failed to write row") + row.ReportTime = reportTs + if slices.Index(systemDBs, row.DBName) >= 0 { + continue } + resultReport.Println(row) } } return nil