diff --git a/dbm-services/mysql/db-tools/mysql-crond/mysql_crond_keep_alive.sh b/dbm-services/mysql/db-tools/mysql-crond/mysql_crond_keep_alive.sh index 9ae3420a27..59f956e975 100644 --- a/dbm-services/mysql/db-tools/mysql-crond/mysql_crond_keep_alive.sh +++ b/dbm-services/mysql/db-tools/mysql-crond/mysql_crond_keep_alive.sh @@ -13,7 +13,8 @@ function mysql_crond_is_alive() { print_log "mysql-crond process found, pid=${PID}" curl -XGET http://127.0.0.1:9999/entries 1>/dev/null 2>&1 if [ $? -eq 0 ];then - print_log "connect mysql-crond success" + echo $PID > /home/mysql/mysql-crond/mysql-crond.pid + print_log "connect mysql-crond $PID success" return 0 else print_log "connect mysql-crond failed, kill pid=${PID}" diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go index 867e724805..d89245a0f3 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go @@ -10,8 +10,10 @@ package mainloop import ( + "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" "fmt" "log/slog" + "path/filepath" "slices" "strings" @@ -21,6 +23,7 @@ import ( "dbm-services/mysql/db-tools/mysql-monitor/pkg/utils" _ "github.com/go-sql-driver/mysql" // mysql TODO + "github.com/juju/fslock" "github.com/pkg/errors" "github.com/spf13/viper" ) @@ -36,6 +39,22 @@ func Run(hardcode bool) error { slog.Info("main loop", slog.String("items", strings.Join(iNames, ","))) slog.Info("main loop", slog.Bool("hardcode", hardcode)) + lockFileName := fmt.Sprintf("%d-%s.lock", config.MonitorConfig.Port, strings.Join(iNames, ".")) + lockFilePath := filepath.Join(cst.MySQLMonitorInstallPath, lockFileName) + + slog.Info("man loop", slog.String("lockFilePath", lockFilePath)) + lk := fslock.New(lockFilePath) + err := lk.TryLock() + if err != nil { + slog.Error("main loop", + slog.String("error", err.Error())) + utils.SendMonitorEvent("db-hang", lockFileName) + return errors.Wrapf(err, "main loop lock file %s failed, may be last round not finish", lockFilePath) + } + defer func() { + _ = lk.Unlock() + }() + if hardcode && slices.Index(iNames, config.HeartBeatName) >= 0 { utils.SendMonitorMetrics(config.HeartBeatName, 1, nil) }