newlog: fix file naming for logs

Revert the file naming inside devUpload and keepSentQueue directories to
the original format `dev.log.<timestamp>.gz` to keep the compatibility
with previous versions of the newlogd.

Signed-off-by: Paul Gaiduk <[email protected]>

docs/LOGGING.md

@@ -53,7 +53,8 @@ When the above log files are closed either due to size or time limit reached, th
 
 The size of the gzip file is limited to 50 KBytes due to the northbound queueing configuration.
 If the compressed file is larger than the limit, it will be split and compressed into two separate gzip files.
-The gzip filename is encoded with current timestamp in Unix milliseconds, such as 'dev.log.upload.1600831551491.gz' for device log, and with timestamp and application UUID such as 'app.62195aa9-7db4-4ac0-86d3-d8abe0ff0ea9.log.1599186248917.gz' for application logs.
+The gzip filename is encoded with current timestamp in Unix milliseconds, such as 'dev.log.1600831551491.gz' for device log, and with timestamp and application UUID such as 'app.62195aa9-7db4-4ac0-86d3-d8abe0ff0ea9.log.1599186248917.gz' for application logs.
+The words "keep" or "upload" are not part of the gzip filename, but the directory name where the gzip file is stored indicates the purpose of the log file.
 The metadata such as device-UUID, the image partition, and image version or app Name for application are encoded as part of the gzip metadata header along with the gzip file.
 
 Upon the device restart, any unfinished temporary log files of previous life left in /persist/newlog/collect directory will be first moved and compressed by newlogd daemon into their upload gzip directories before any current log events are written onto the disk.
@@ -115,7 +116,7 @@ The uploading is controlled on a scheduled timer. When the timer fires, the "log
 
 The "loguploader" collects stats of round-trip delay, controller CPU load percentage and log batch processing time. The current EVE implementation does not use those stats in calculating the uploading timer values.
 
-The already uploaded gzip files with app logs are moved to /persist/newlog/keepSentQueue directory or removed in case of dev.log.upload files.
+The already uploaded gzip files with app logs are moved to /persist/newlog/keepSentQueue directory or removed in case of dev.log files.
 This directory and together with 'collect', 'appUpload', 'devUpload' directories form a circular buffer and will be kept up to the quota limit (default is 2 Gbytes and can be changed by user config-item).
 
 To prevent the log messages grow without bounds over time, the 'failedUpload' directory will only keep up to 1000 gzip files, each with maximum of 50K, to be under 50M in the directory. The '/persist' partition space is monitored, and if the available space is under 100M, the 'newlogd' will kick in the gzip file recycle operation just as the controller uplink is unreachable.

pkg/edgeview/src/edgeview_test.go

@@ -28,7 +28,7 @@ func TestWalkLogDirs(t *testing.T) {
 	g.Expect(foundFiles).To(HaveLen(1), "expected exactly one file to be found")
 
 	expected := logfiletime{
-		filepath: path.Join(newlogDir, "keepSentQueue/dev.log.keep.1731491932618.gz"),
+		filepath: path.Join(newlogDir, "keepSentQueue/dev.log.1731491932618.gz"),
 		filesec:  1731491932,
 	}
 	g.Expect(foundFiles[0]).To(Equal(expected))

pkg/newlog/cmd/newlogd.go

@@ -1168,46 +1168,36 @@ func checkKeepQuota() {
 }
 
 func getOldestLog() (*logs.LogEntry, error) {
-	// Compile a regex to match the log file pattern and extract the timestamp
-	re := regexp.MustCompile(`^dev\.log\.keep\.(\d+)\.gz$`)
-
 	// Read the directory and filter log files
 	files, err := os.ReadDir(keepSentDir)
 	if err != nil {
 		return nil, fmt.Errorf("error reading directory: %w", err)
 	}
 
-	type logFile struct {
-		name      string
-		timestamp string
-	}
+	oldestLogFileName := ""
+	oldestLogFileTimestamp := time.Now()
 
-	var devLogFiles []logFile
 	for _, file := range files {
 		if file.IsDir() {
 			continue
 		}
-		matches := re.FindStringSubmatch(file.Name())
-		if matches != nil {
-			devLogFiles = append(devLogFiles, logFile{
-				name:      file.Name(),
-				timestamp: matches[1],
-			})
+		timestamp, err := types.GetTimestampFromGzipName(file.Name())
+		if err != nil {
+			continue
+		}
+		if timestamp.Before(oldestLogFileTimestamp) {
+			oldestLogFileTimestamp = timestamp
+			oldestLogFileName = file.Name()
 		}
 	}
 
-	if len(devLogFiles) == 0 {
+	if oldestLogFileName == "" {
 		log.Function("getLatestLog: no log files found.")
 		return nil, nil
 	}
 
-	// Sort the log files by timestamp (ascending order)
-	sort.Slice(devLogFiles, func(i, j int) bool {
-		return devLogFiles[i].timestamp < devLogFiles[j].timestamp
-	})
-
 	// Open the oldest log file
-	oldestFile := filepath.Join(keepSentDir, devLogFiles[0].name)
+	oldestFile := filepath.Join(keepSentDir, oldestLogFileName)
 	file, err := os.Open(oldestFile)
 	if err != nil {
 		return nil, fmt.Errorf("error opening file: %w", err)
@@ -1447,11 +1437,7 @@ func gzipFileNameGet(isApp bool, timeNum int, dirName, appUUID string, notUpload
 		}
 		outfileName = appPref + appUUID + types.AppSuffix + strconv.Itoa(timeNum) + ".gz"
 	} else {
-		if notUpload {
-			outfileName = devPrefixKeep + strconv.Itoa(timeNum) + ".gz"
-		} else {
-			outfileName = devPrefixUpload + strconv.Itoa(timeNum) + ".gz"
-		}
+		outfileName = devPrefix + strconv.Itoa(timeNum) + ".gz"
 	}
 	return dirName + "/" + outfileName
 }

pkg/newlog/cmd/newlogd_test.go

@@ -8,11 +8,16 @@ import (
 	"testing"
 	"time"
 
+	"github.com/lf-edge/eve/pkg/pillar/agentlog"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	. "github.com/onsi/gomega"
 )
 
+func init() {
+	logger, log = agentlog.Init(agentName)
+}
 func TestGzipParsing(t *testing.T) {
+	t.Parallel()
 	g := NewWithT(t)
 
 	// Test the gzip parsing function
@@ -31,13 +36,14 @@ func TestGzipParsing(t *testing.T) {
 }
 
 func TestGetTimestampFromGzipName(t *testing.T) {
+	t.Parallel()
 	g := NewWithT(t)
 
 	comparisonMap := map[string]time.Time{
 		"app.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.log.1731935033496.gz": time.Unix(0, 1731935033496*int64(time.Millisecond)),
-		"dev.log.keep.1731491904032.gz":                                 time.Unix(0, 1731491904032*int64(time.Millisecond)),
-		"dev.log.keep.1731491932618.gz":                                 time.Unix(0, 1731491932618*int64(time.Millisecond)),
-		"dev.log.keep.1731491940142.gz":                                 time.Unix(0, 1731491940142*int64(time.Millisecond)),
+		"dev.log.1731491904032.gz":                                      time.Unix(0, 1731491904032*int64(time.Millisecond)),
+		"dev.log.1731491932618.gz":                                      time.Unix(0, 1731491932618*int64(time.Millisecond)),
+		"dev.log.1731491940142.gz":                                      time.Unix(0, 1731491940142*int64(time.Millisecond)),
 	}
 
 	keepSentDir = "../testdata/keepSentQueue"
@@ -55,6 +61,7 @@ func TestGetTimestampFromGzipName(t *testing.T) {
 }
 
 func TestFindMovePrevLogFiles(t *testing.T) {
+	t.Parallel()
 	g := NewWithT(t)
 
 	collectDir = "../testdata/collect"
@@ -106,6 +113,7 @@ func TestFindMovePrevLogFiles(t *testing.T) {
 }
 
 func TestGetFileInfo(t *testing.T) {
+	t.Parallel()
 	g := NewWithT(t)
 
 	tests := []struct {
@@ -158,6 +166,7 @@ func TestGetFileInfo(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
 			dirName, appuuid := getFileInfo(tt.fileChanInfo)
 			g.Expect(dirName).To(Equal(tt.expectedDir))
 			g.Expect(appuuid).To(Equal(tt.expectedAppID))