From 8e46902c3eb3ee94828897127bfdb80bd919ffbf Mon Sep 17 00:00:00 2001 From: Paul Gaiduk Date: Mon, 16 Dec 2024 20:19:49 +0100 Subject: [PATCH] newlog: fix file naming for logs Revert the file naming inside devUpload and keepSentQueue directories to the original format `dev.log..gz` to keep the compatibility with previous versions of the newlogd. Signed-off-by: Paul Gaiduk --- docs/LOGGING.md | 5 ++- pkg/edgeview/src/edgeview_test.go | 2 +- pkg/newlog/cmd/newlogd.go | 38 ++++++------------ pkg/newlog/cmd/newlogd_test.go | 15 +++++-- ...1491904032.gz => dev.log.1731491904032.gz} | Bin ...1491932618.gz => dev.log.1731491932618.gz} | Bin ...1491940142.gz => dev.log.1731491940142.gz} | Bin 7 files changed, 28 insertions(+), 32 deletions(-) rename pkg/newlog/testdata/keepSentQueue/{dev.log.keep.1731491904032.gz => dev.log.1731491904032.gz} (100%) rename pkg/newlog/testdata/keepSentQueue/{dev.log.keep.1731491932618.gz => dev.log.1731491932618.gz} (100%) rename pkg/newlog/testdata/keepSentQueue/{dev.log.keep.1731491940142.gz => dev.log.1731491940142.gz} (100%) diff --git a/docs/LOGGING.md b/docs/LOGGING.md index d435fda348..87d5a167ef 100644 --- a/docs/LOGGING.md +++ b/docs/LOGGING.md @@ -53,7 +53,8 @@ When the above log files are closed either due to size or time limit reached, th The size of the gzip file is limited to 50 KBytes due to the northbound queueing configuration. If the compressed file is larger than the limit, it will be split and compressed into two separate gzip files. -The gzip filename is encoded with current timestamp in Unix milliseconds, such as 'dev.log.upload.1600831551491.gz' for device log, and with timestamp and application UUID such as 'app.62195aa9-7db4-4ac0-86d3-d8abe0ff0ea9.log.1599186248917.gz' for application logs. +The gzip filename is encoded with current timestamp in Unix milliseconds, such as 'dev.log.1600831551491.gz' for device log, and with timestamp and application UUID such as 'app.62195aa9-7db4-4ac0-86d3-d8abe0ff0ea9.log.1599186248917.gz' for application logs. +The words "keep" or "upload" are not part of the gzip filename, but the directory name where the gzip file is stored indicates the purpose of the log file. The metadata such as device-UUID, the image partition, and image version or app Name for application are encoded as part of the gzip metadata header along with the gzip file. Upon the device restart, any unfinished temporary log files of previous life left in /persist/newlog/collect directory will be first moved and compressed by newlogd daemon into their upload gzip directories before any current log events are written onto the disk. @@ -115,7 +116,7 @@ The uploading is controlled on a scheduled timer. When the timer fires, the "log The "loguploader" collects stats of round-trip delay, controller CPU load percentage and log batch processing time. The current EVE implementation does not use those stats in calculating the uploading timer values. -The already uploaded gzip files with app logs are moved to /persist/newlog/keepSentQueue directory or removed in case of dev.log.upload files. +The already uploaded gzip files with app logs are moved to /persist/newlog/keepSentQueue directory or removed in case of dev.log files. This directory and together with 'collect', 'appUpload', 'devUpload' directories form a circular buffer and will be kept up to the quota limit (default is 2 Gbytes and can be changed by user config-item). To prevent the log messages grow without bounds over time, the 'failedUpload' directory will only keep up to 1000 gzip files, each with maximum of 50K, to be under 50M in the directory. The '/persist' partition space is monitored, and if the available space is under 100M, the 'newlogd' will kick in the gzip file recycle operation just as the controller uplink is unreachable. diff --git a/pkg/edgeview/src/edgeview_test.go b/pkg/edgeview/src/edgeview_test.go index 2de6943970..5f6a25d6d6 100644 --- a/pkg/edgeview/src/edgeview_test.go +++ b/pkg/edgeview/src/edgeview_test.go @@ -28,7 +28,7 @@ func TestWalkLogDirs(t *testing.T) { g.Expect(foundFiles).To(HaveLen(1), "expected exactly one file to be found") expected := logfiletime{ - filepath: path.Join(newlogDir, "keepSentQueue/dev.log.keep.1731491932618.gz"), + filepath: path.Join(newlogDir, "keepSentQueue/dev.log.1731491932618.gz"), filesec: 1731491932, } g.Expect(foundFiles[0]).To(Equal(expected)) diff --git a/pkg/newlog/cmd/newlogd.go b/pkg/newlog/cmd/newlogd.go index e7841e5a96..4641286a85 100644 --- a/pkg/newlog/cmd/newlogd.go +++ b/pkg/newlog/cmd/newlogd.go @@ -1168,46 +1168,36 @@ func checkKeepQuota() { } func getOldestLog() (*logs.LogEntry, error) { - // Compile a regex to match the log file pattern and extract the timestamp - re := regexp.MustCompile(`^dev\.log\.keep\.(\d+)\.gz$`) - // Read the directory and filter log files files, err := os.ReadDir(keepSentDir) if err != nil { return nil, fmt.Errorf("error reading directory: %w", err) } - type logFile struct { - name string - timestamp string - } + oldestLogFileName := "" + oldestLogFileTimestamp := time.Now() - var devLogFiles []logFile for _, file := range files { if file.IsDir() { continue } - matches := re.FindStringSubmatch(file.Name()) - if matches != nil { - devLogFiles = append(devLogFiles, logFile{ - name: file.Name(), - timestamp: matches[1], - }) + timestamp, err := types.GetTimestampFromGzipName(file.Name()) + if err != nil { + continue + } + if timestamp.Before(oldestLogFileTimestamp) { + oldestLogFileTimestamp = timestamp + oldestLogFileName = file.Name() } } - if len(devLogFiles) == 0 { + if oldestLogFileName == "" { log.Function("getLatestLog: no log files found.") return nil, nil } - // Sort the log files by timestamp (ascending order) - sort.Slice(devLogFiles, func(i, j int) bool { - return devLogFiles[i].timestamp < devLogFiles[j].timestamp - }) - // Open the oldest log file - oldestFile := filepath.Join(keepSentDir, devLogFiles[0].name) + oldestFile := filepath.Join(keepSentDir, oldestLogFileName) file, err := os.Open(oldestFile) if err != nil { return nil, fmt.Errorf("error opening file: %w", err) @@ -1447,11 +1437,7 @@ func gzipFileNameGet(isApp bool, timeNum int, dirName, appUUID string, notUpload } outfileName = appPref + appUUID + types.AppSuffix + strconv.Itoa(timeNum) + ".gz" } else { - if notUpload { - outfileName = devPrefixKeep + strconv.Itoa(timeNum) + ".gz" - } else { - outfileName = devPrefixUpload + strconv.Itoa(timeNum) + ".gz" - } + outfileName = devPrefix + strconv.Itoa(timeNum) + ".gz" } return dirName + "/" + outfileName } diff --git a/pkg/newlog/cmd/newlogd_test.go b/pkg/newlog/cmd/newlogd_test.go index ac26cdff22..391b1f7e40 100644 --- a/pkg/newlog/cmd/newlogd_test.go +++ b/pkg/newlog/cmd/newlogd_test.go @@ -8,11 +8,16 @@ import ( "testing" "time" + "github.com/lf-edge/eve/pkg/pillar/agentlog" "github.com/lf-edge/eve/pkg/pillar/types" . "github.com/onsi/gomega" ) +func init() { + logger, log = agentlog.Init(agentName) +} func TestGzipParsing(t *testing.T) { + t.Parallel() g := NewWithT(t) // Test the gzip parsing function @@ -31,13 +36,14 @@ func TestGzipParsing(t *testing.T) { } func TestGetTimestampFromGzipName(t *testing.T) { + t.Parallel() g := NewWithT(t) comparisonMap := map[string]time.Time{ "app.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.log.1731935033496.gz": time.Unix(0, 1731935033496*int64(time.Millisecond)), - "dev.log.keep.1731491904032.gz": time.Unix(0, 1731491904032*int64(time.Millisecond)), - "dev.log.keep.1731491932618.gz": time.Unix(0, 1731491932618*int64(time.Millisecond)), - "dev.log.keep.1731491940142.gz": time.Unix(0, 1731491940142*int64(time.Millisecond)), + "dev.log.1731491904032.gz": time.Unix(0, 1731491904032*int64(time.Millisecond)), + "dev.log.1731491932618.gz": time.Unix(0, 1731491932618*int64(time.Millisecond)), + "dev.log.1731491940142.gz": time.Unix(0, 1731491940142*int64(time.Millisecond)), } keepSentDir = "../testdata/keepSentQueue" @@ -55,6 +61,7 @@ func TestGetTimestampFromGzipName(t *testing.T) { } func TestFindMovePrevLogFiles(t *testing.T) { + t.Parallel() g := NewWithT(t) collectDir = "../testdata/collect" @@ -106,6 +113,7 @@ func TestFindMovePrevLogFiles(t *testing.T) { } func TestGetFileInfo(t *testing.T) { + t.Parallel() g := NewWithT(t) tests := []struct { @@ -158,6 +166,7 @@ func TestGetFileInfo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() dirName, appuuid := getFileInfo(tt.fileChanInfo) g.Expect(dirName).To(Equal(tt.expectedDir)) g.Expect(appuuid).To(Equal(tt.expectedAppID)) diff --git a/pkg/newlog/testdata/keepSentQueue/dev.log.keep.1731491904032.gz b/pkg/newlog/testdata/keepSentQueue/dev.log.1731491904032.gz similarity index 100% rename from pkg/newlog/testdata/keepSentQueue/dev.log.keep.1731491904032.gz rename to pkg/newlog/testdata/keepSentQueue/dev.log.1731491904032.gz diff --git a/pkg/newlog/testdata/keepSentQueue/dev.log.keep.1731491932618.gz b/pkg/newlog/testdata/keepSentQueue/dev.log.1731491932618.gz similarity index 100% rename from pkg/newlog/testdata/keepSentQueue/dev.log.keep.1731491932618.gz rename to pkg/newlog/testdata/keepSentQueue/dev.log.1731491932618.gz diff --git a/pkg/newlog/testdata/keepSentQueue/dev.log.keep.1731491940142.gz b/pkg/newlog/testdata/keepSentQueue/dev.log.1731491940142.gz similarity index 100% rename from pkg/newlog/testdata/keepSentQueue/dev.log.keep.1731491940142.gz rename to pkg/newlog/testdata/keepSentQueue/dev.log.1731491940142.gz