fix(mysql): mysql-crond pid文件和实际pid不一致的问题 #8104

TencentBlueKing · Nov 22, 2024 · 33ee32d · 33ee32d
1 parent 0c40355
commit 33ee32d
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 1 deletion.
diff --git a/dbm-services/mysql/db-tools/mysql-crond/mysql_crond_keep_alive.sh b/dbm-services/mysql/db-tools/mysql-crond/mysql_crond_keep_alive.sh
@@ -13,7 +13,8 @@ function mysql_crond_is_alive() {
                         print_log "mysql-crond process found, pid=${PID}"
                         curl -XGET http://127.0.0.1:9999/entries 1>/dev/null 2>&1
                         if [ $? -eq 0 ];then
-                                print_log "connect mysql-crond success"
+                                echo $PID > /home/mysql/mysql-crond/mysql-crond.pid
+                                print_log "connect mysql-crond $PID success"
                                 return 0
                         else
                                 print_log "connect mysql-crond failed, kill pid=${PID}"

diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go
@@ -10,8 +10,10 @@
 package mainloop
 
 import (
+	"dbm-services/mysql/db-tools/dbactuator/pkg/core/cst"
 	"fmt"
 	"log/slog"
+	"path/filepath"
 	"slices"
 	"strings"
 
@@ -21,6 +23,7 @@ import (
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/utils"
 
 	_ "github.com/go-sql-driver/mysql" // mysql TODO
+	"github.com/juju/fslock"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 )
@@ -36,6 +39,22 @@ func Run(hardcode bool) error {
 	slog.Info("main loop", slog.String("items", strings.Join(iNames, ",")))
 	slog.Info("main loop", slog.Bool("hardcode", hardcode))
 
+	lockFileName := fmt.Sprintf("%d-%s.lock", config.MonitorConfig.Port, strings.Join(iNames, "."))
+	lockFilePath := filepath.Join(cst.MySQLMonitorInstallPath, lockFileName)
+
+	slog.Info("man loop", slog.String("lockFilePath", lockFilePath))
+	lk := fslock.New(lockFilePath)
+	err := lk.TryLock()
+	if err != nil {
+		slog.Error("main loop",
+			slog.String("error", err.Error()))
+		utils.SendMonitorEvent("db-hang", lockFileName)
+		return errors.Wrapf(err, "main loop lock file %s failed, may be last round not finish", lockFilePath)
+	}
+	defer func() {
+		_ = lk.Unlock()
+	}()
+
 	if hardcode && slices.Index(iNames, config.HeartBeatName) >= 0 {
 		utils.SendMonitorMetrics(config.HeartBeatName, 1, nil)
 	}