feat(mysql): mysql-monitor增加disable子命令

dbm-services/mysql/db-tools/mysql-monitor/README.md

@@ -15,6 +15,21 @@
 ## _clean_
 * 执行 `mysql-monitor -c runtime.yaml clean` 会删除所有相关的 `mysql-crond entry`
 
+* 会触发 `监控心跳丢失` 的告警
+* 一般只用于下架场景
+* 如果临时停止监控需求, 用下面的 `disable-all`
+
+## _disable-all_
+`mysql-monitor disable-all -c monitor-config_20000.yaml --staff somebody --with-db-up`
+* 保留 `监控心跳`
+* 停掉包括`db-up` 在内的所有监控项
+
+如果不使用 `--with-db-up`, 则会保留 `db-up` 监控项
+
+不修改任何配置文件, _disable_ 不会持久化, 可以随时使用上面提到的 _reschedule_ 恢复回来
+
+
+
 ## 硬编码项
 目前有两个硬编码项
 1. 执行心跳

dbm-services/mysql/db-tools/mysql-monitor/cmd/reschedule_items.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	ma "dbm-services/mysql/db-tools/mysql-crond/api"
+	"dbm-services/mysql/db-tools/mysql-monitor/pkg/config"
+
+	"golang.org/x/exp/slog"
+)
+
+func reschedule(configFileDir, configFileName, staff string) error {
+	manager := ma.NewManager(config.MonitorConfig.ApiUrl)
+	entries, err := manager.Entries()
+	if err != nil {
+		slog.Error("reschedule list entries", err)
+		return err
+	}
+
+	for _, entry := range entries {
+		if strings.HasPrefix(
+			entry.Job.Name,
+			fmt.Sprintf("mysql-monitor-%d", config.MonitorConfig.Port),
+		) {
+			eid, err := manager.Delete(entry.Job.Name, true)
+			if err != nil {
+				slog.Error(
+					"reschedule delete entry", err,
+					slog.String("name", entry.Job.Name),
+				)
+				return err
+			}
+			slog.Info(
+				"reschedule delete entry",
+				slog.String("name", entry.Job.Name),
+				slog.Int("ID", eid),
+			)
+		}
+	}
+
+	var hardCodeItems []*config.MonitorItem
+	itemGroups := make(map[string][]*config.MonitorItem)
+	for _, ele := range config.ItemsConfig {
+		// 硬编码监控项先排除掉
+		if ele.Name == "db-up" || ele.Name == config.HeartBeatName {
+			if ele.IsEnable() {
+				hardCodeItems = append(hardCodeItems, ele)
+			}
+			continue
+		}
+
+		if ele.IsEnable() && ele.IsMatchMachineType() && ele.IsMatchRole() {
+			var key string
+
+			if ele.Schedule == nil {
+				key = config.MonitorConfig.DefaultSchedule
+			} else {
+				key = *ele.Schedule
+			}
+
+			if _, ok := itemGroups[key]; !ok {
+				itemGroups[key] = []*config.MonitorItem{}
+			}
+			itemGroups[key] = append(itemGroups[key], ele)
+		}
+	}
+
+	for k, v := range itemGroups {
+		var itemNames []string
+		for _, j := range v {
+			itemNames = append(itemNames, j.Name)
+		}
+		args := []string{
+			"run",
+			"--items", strings.Join(itemNames, ","),
+			"-c", configFileName, // use WorkDir
+		}
+		eid, err := manager.CreateOrReplace(
+			ma.JobDefine{
+				Name:     fmt.Sprintf("mysql-monitor-%d-%s", config.MonitorConfig.Port, k),
+				Command:  executable,
+				Args:     args,
+				Schedule: k,
+				Creator:  staff, //viper.GetString("staff"),
+				Enable:   true,
+				WorkDir:  configFileDir,
+			}, true,
+		)
+		if err != nil {
+			slog.Error("reschedule add entry", err)
+			return err
+		}
+		slog.Info("reschedule add entry", slog.Int("entry id", eid))
+	}
+
+	// 注册 hardcode
+	for _, j := range hardCodeItems {
+		args := []string{
+			"hardcode-run",
+			"--items", j.Name, //strings.Join(itemNames, ","),
+			"-c", configFileName,
+		}
+
+		eid, err := manager.CreateOrReplace(
+			ma.JobDefine{
+				Name: fmt.Sprintf(
+					"mysql-monitor-%d-hardcode-%s", config.MonitorConfig.Port, j.Name),
+				Command:  executable,
+				Args:     args,
+				Schedule: config.HardCodeSchedule,
+				Creator:  staff, //viper.GetString("staff"),
+				Enable:   true,
+				WorkDir:  configFileDir,
+			}, true,
+		)
+		if err != nil {
+			slog.Error("reschedule add hardcode entry", err)
+			return err
+		}
+		slog.Info("reschedule add hardcode entry", slog.Int("entry id", eid))
+	}
+
+	return nil
+}

dbm-services/mysql/db-tools/mysql-monitor/cmd/subcmd_disable_all.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/exp/slog"
+
+	"dbm-services/mysql/db-tools/mysql-monitor/pkg/config"
+)
+
+var subCmdDisableAll = &cobra.Command{
+	Use:   "disable-all",
+	Short: "disable-all items",
+	Long:  "disable-all items",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		configPath := viper.GetString("disable-config")
+		if !filepath.IsAbs(configPath) {
+			cwd, err := os.Getwd()
+			if err != nil {
+				slog.Error("disable-all get config abs path", err)
+				return err
+			}
+			configPath = filepath.Join(cwd, configPath)
+		}
+		configFileDir, configFileName := filepath.Split(configPath)
+
+		err := config.InitConfig(configPath)
+		if err != nil {
+			return err
+		}
+		initLogger(config.MonitorConfig.Log)
+
+		emptyItemsConfig, err := os.CreateTemp("/tmp", "empty-items.yaml")
+		if err != nil {
+			slog.Error("disable-all create empty items config", slog.String("error", err.Error()))
+			return err
+		}
+		defer func() {
+			_ = emptyItemsConfig.Close()
+			_ = os.Remove(emptyItemsConfig.Name())
+		}()
+		slog.Info("disable-all create empty items config success")
+
+		config.MonitorConfig.ItemsConfigFile = emptyItemsConfig.Name()
+
+		err = config.LoadMonitorItemsConfig()
+		if err != nil {
+			slog.Error("disable-all load items", err)
+			return err
+		}
+
+		disableDbUp := viper.GetBool("with-db-up")
+		if !disableDbUp {
+			config.InjectMonitorDbUpItem()
+		}
+		config.InjectMonitorHeartBeatItem()
+
+		slog.Info("disable-all",
+			slog.String("staff", viper.GetString("staff")))
+		err = reschedule(configFileDir, configFileName, viper.GetString("disable-staff"))
+		if err != nil {
+			slog.Error("disable-all sub-cmd", slog.String("error", err.Error()))
+			return err
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	subCmdDisableAll.PersistentFlags().StringP("config", "c", "", "config file")
+	_ = subCmdDisableAll.MarkPersistentFlagRequired("config")
+	_ = viper.BindPFlag("disable-config", subCmdDisableAll.PersistentFlags().Lookup("config"))
+
+	subCmdDisableAll.PersistentFlags().StringP("staff", "", "", "staff name")
+	_ = subCmdDisableAll.MarkPersistentFlagRequired("staff")
+	_ = viper.BindPFlag("disable-staff", subCmdDisableAll.PersistentFlags().Lookup("staff"))
+
+	subCmdDisableAll.PersistentFlags().BoolP("with-db-up", "", false, "also disable db-up")
+	_ = viper.BindPFlag("with-db-up", subCmdDisableAll.PersistentFlags().Lookup("with-db-up"))
+
+	rootCmd.AddCommand(subCmdDisableAll)
+}

dbm-services/mysql/db-tools/mysql-monitor/cmd/subcmd_reschedule.go

@@ -1,12 +1,9 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
-	ma "dbm-services/mysql/db-tools/mysql-crond/api"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/config"
 
 	"github.com/spf13/cobra"
@@ -46,123 +43,21 @@ var subCmdReschedule = &cobra.Command{
 			return err
 		}
 
-		config.InjectHardCodeItem()
+		config.InjectMonitorDbUpItem()
+		config.InjectMonitorHeartBeatItem()
 
 		err = config.WriteMonitorItemsBack()
 		if err != nil {
 			slog.Error("reschedule write back items", err)
 			return err
 		}
 
-		manager := ma.NewManager(config.MonitorConfig.ApiUrl)
-		entries, err := manager.Entries()
+		err = reschedule(configFileDir, configFileName, viper.GetString("reschedule-staff"))
 		if err != nil {
-			slog.Error("reschedule list entries", err)
+			slog.Error("reschedule sub-cmd", slog.String("error", err.Error()))
 			return err
 		}
 
-		for _, entry := range entries {
-			if strings.HasPrefix(
-				entry.Job.Name,
-				fmt.Sprintf("mysql-monitor-%d", config.MonitorConfig.Port),
-			) {
-				eid, err := manager.Delete(entry.Job.Name, true)
-				if err != nil {
-					slog.Error(
-						"reschedule delete entry", err,
-						slog.String("name", entry.Job.Name),
-					)
-					return err
-				}
-				slog.Info(
-					"reschedule delete entry",
-					slog.String("name", entry.Job.Name),
-					slog.Int("ID", eid),
-				)
-			}
-		}
-
-		var hardCodeItems []*config.MonitorItem
-		itemGroups := make(map[string][]*config.MonitorItem)
-		for _, ele := range config.ItemsConfig {
-			// 硬编码监控项先排除掉
-			if ele.Name == "db-up" || ele.Name == config.HeartBeatName {
-				if ele.IsEnable() {
-					hardCodeItems = append(hardCodeItems, ele)
-				}
-				continue
-			}
-
-			if ele.IsEnable() && ele.IsMatchMachineType() && ele.IsMatchRole() {
-				var key string
-
-				if ele.Schedule == nil {
-					key = config.MonitorConfig.DefaultSchedule
-				} else {
-					key = *ele.Schedule
-				}
-
-				if _, ok := itemGroups[key]; !ok {
-					itemGroups[key] = []*config.MonitorItem{}
-				}
-				itemGroups[key] = append(itemGroups[key], ele)
-			}
-		}
-
-		for k, v := range itemGroups {
-			var itemNames []string
-			for _, j := range v {
-				itemNames = append(itemNames, j.Name)
-			}
-			args := []string{
-				"run",
-				"--items", strings.Join(itemNames, ","),
-				"-c", configFileName, // use WorkDir
-			}
-			eid, err := manager.CreateOrReplace(
-				ma.JobDefine{
-					Name:     fmt.Sprintf("mysql-monitor-%d-%s", config.MonitorConfig.Port, k),
-					Command:  executable,
-					Args:     args,
-					Schedule: k,
-					Creator:  viper.GetString("staff"),
-					Enable:   true,
-					WorkDir:  configFileDir,
-				}, true,
-			)
-			if err != nil {
-				slog.Error("reschedule add entry", err)
-				return err
-			}
-			slog.Info("reschedule add entry", slog.Int("entry id", eid))
-		}
-
-		// 注册 hardcode
-		var itemNames []string
-		for _, j := range hardCodeItems {
-			itemNames = append(itemNames, j.Name)
-		}
-		args = []string{
-			"hardcode-run",
-			"--items", strings.Join(itemNames, ","),
-			"-c", configPath,
-		}
-		eid, err := manager.CreateOrReplace(
-			ma.JobDefine{
-				Name:     fmt.Sprintf("mysql-monitor-%d-hardcode", config.MonitorConfig.Port),
-				Command:  executable,
-				Args:     args,
-				Schedule: config.HardCodeSchedule,
-				Creator:  viper.GetString("staff"),
-				Enable:   true,
-			}, true,
-		)
-		if err != nil {
-			slog.Error("reschedule add hardcode entry", err)
-			return err
-		}
-		slog.Info("reschedule add hardcode entry", slog.Int("entry id", eid))
-
 		return nil
 	},
 }
@@ -174,7 +69,7 @@ func init() {
 
 	subCmdReschedule.PersistentFlags().StringP("staff", "", "", "staff name")
 	_ = subCmdReschedule.MarkPersistentFlagRequired("staff")
-	_ = viper.BindPFlag("staff", subCmdReschedule.PersistentFlags().Lookup("staff"))
+	_ = viper.BindPFlag("reschedule-staff", subCmdReschedule.PersistentFlags().Lookup("staff"))
 
 	rootCmd.AddCommand(subCmdReschedule)
 }