Merge pull request #42 from sensu/echlebek/fix-token-too-long

Don't use bufio.Scanner to read the log

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15.x
+          go-version: 1.21.x
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:

.github/workflows/static-check.yml

@@ -0,0 +1,23 @@
+name: static-check
+
+on:
+  pull_request:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - main
+
+jobs:
+  staticcheck:
+    name: staticcheck (project)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - uses: dominikh/[email protected]
+        with:
+          version: "2023.1.1"
+env:
+  GO_VERSION: 1.21.5

.github/workflows/test.yml

@@ -12,10 +12,10 @@ jobs:
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
-    - name: Set up Go 1.14
+    - name: Set up Go 1.21
       uses: actions/setup-go@v1
       with:
-        go-version: 1.14
+        go-version: 1.21
       id: go
     - name: Test
       run: go test -v ./...

analyzer.go

@@ -4,8 +4,8 @@ import (
 	"bufio"
 	"context"
 	"io"
-	"io/ioutil"
 	"sync"
+	"sync/atomic"
 )
 
 const bufSize = 1000
@@ -44,17 +44,17 @@ func (d *DiscardWriter) ReadFrom(r io.Reader) (int64, error) {
 }
 
 func NewDiscardWriter() *DiscardWriter {
-	discard := ioutil.Discard.(discardInterface)
+	discard := io.Discard.(discardInterface)
 	return &DiscardWriter{discardWriter: discard}
 }
 
 type AnalyzerFunc func([]byte) *Result
 
 type Result struct {
-	Path    string `json:"path"`
-	Match   string `json:"match,omitempty"`
-	Err     error  `json:"error,omitempty"`
-	Offset  int64  `json:"offset"`
+	Path   string `json:"path"`
+	Match  string `json:"match,omitempty"`
+	Err    error  `json:"error,omitempty"`
+	Offset int64  `json:"offset"`
 }
 
 type LineMsg struct {
@@ -77,46 +77,37 @@ func (a *Analyzer) Go(ctx context.Context) <-chan Result {
 }
 
 func (a *Analyzer) BytesRead() int64 {
-	a.wg.Wait()
-	return a.bytesRead
+	return atomic.LoadInt64(&a.bytesRead)
 }
 
 func (a *Analyzer) startProducer(ctx context.Context) <-chan LineMsg {
 	logLines := make(chan LineMsg, bufSize)
 	currentOffset := a.Offset
 	reader := bufio.NewReaderSize(a.Log, 32*1024*1024)
-	discard := NewDiscardWriter()
-	teeReader := io.TeeReader(reader, discard)
-	scanner := bufio.NewScanner(teeReader)
-	buf := make([]byte, 0, 64*1024)
-	scanner.Buffer(buf, 1024*1024)
-	var wg sync.WaitGroup
-	wg.Add(1)
 	a.wg.Add(1)
 	go func() {
-		defer wg.Done()
-		for scanner.Scan() {
-			line := scanner.Bytes()
-			// Copy the line, since the scanner can reclaim it
-			lineCopy := make([]byte, len(line))
-			copy(lineCopy, line)
-			select {
-			case <-ctx.Done():
-				close(logLines)
+		defer a.wg.Done()
+		defer close(logLines)
+		for {
+			line, err := reader.ReadBytes('\n')
+			if err != nil && err != io.EOF {
+				fatal("error while scanning log: %s: %s\n", a.Path, err)
+			}
+			atomic.AddInt64(&a.bytesRead, int64(len(line)))
+			if len(line) > 0 {
+				select {
+				case <-ctx.Done():
+					return
+				case logLines <- LineMsg{Line: line, Offset: currentOffset}:
+					currentOffset += int64(len(line))
+				}
+				if err == io.EOF {
+					return
+				}
+			} else {
 				return
-			case logLines <- LineMsg{Line: lineCopy, Offset: currentOffset}:
-				currentOffset += int64(len(line))
 			}
 		}
-		if err := scanner.Err(); err != nil {
-			fatal("error while scanning log: %s :: %s\n", a.Path, err)
-		}
-		close(logLines)
-	}()
-	go func() {
-		defer a.wg.Done()
-		wg.Wait()
-		a.bytesRead = discard.BytesRead
 	}()
 	return logLines
 }

event.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	corev2 "github.com/sensu/sensu-go/api/core/v2"
+	corev2 "github.com/sensu/core/v2"
 	"github.com/sensu/sensu-plugin-sdk/templates"
 )
 

go.mod

@@ -1,17 +1,58 @@
 module github.com/sensu/sensu-check-log
 
-go 1.12
+go 1.21
 
 require (
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/sensu/core/v2 v2.20.0
+	github.com/sensu/sensu-plugin-sdk v0.18.0
+	github.com/stretchr/testify v1.8.4
+)
+
+require (
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/echlebek/timeproxy v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/robfig/cron v1.2.0 // indirect
-	github.com/sensu/sensu-go/api/core/v2 v2.3.0
-	github.com/sensu/sensu-plugin-sdk v0.14.1
-	github.com/stretchr/testify v1.6.0
-	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/uuid v1.4.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/robertkrimen/otto v0.2.1 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sensu/sensu-api-tools v0.2.1 // indirect
+	github.com/sensu/sensu-go/types v0.13.0 // indirect
+	github.com/sensu/sensu-licensing/v2 v2.2.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.18.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.11 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
+	google.golang.org/grpc v1.59.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )