From 52e9da29dc4d6571f2106205dcb9c2c377c887b9 Mon Sep 17 00:00:00 2001 From: ShobhitSingh11 <139750384+ShobhitSingh11@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:53:55 +0530 Subject: [PATCH] fix: [CI-14200]: Trimming new line suffix from Log Line to avoid extra new line space (#314) * fix: [CI-14200]: Trimming new line suffix from Log Line to avoid extra new line space * fix: [CI-14200]: debugging --- api/api.go | 9 +++---- livelog/livelog.go | 40 ++++++++++++++++++------------- livelog/livelog_test.go | 28 +++++++++++++++++++++- pipeline/runtime/common.go | 2 +- pipeline/runtime/step_executor.go | 3 ++- pipeline/state.go | 7 ++++++ 6 files changed, 65 insertions(+), 24 deletions(-) diff --git a/api/api.go b/api/api.go index 10d8bbc2..601f8239 100644 --- a/api/api.go +++ b/api/api.go @@ -162,10 +162,11 @@ type ( } LogConfig struct { - AccountID string `json:"account_id,omitempty"` - IndirectUpload bool `json:"indirect_upload,omitempty"` // Whether to directly upload via signed link or using log service - URL string `json:"url,omitempty"` - Token string `json:"token,omitempty"` + AccountID string `json:"account_id,omitempty"` + IndirectUpload bool `json:"indirect_upload,omitempty"` // Whether to directly upload via signed link or using log service + URL string `json:"url,omitempty"` + Token string `json:"token,omitempty"` + TrimNewLineSuffix bool `json:"trim_new_line_suffix,omitempty"` } TIConfig struct { diff --git a/livelog/livelog.go b/livelog/livelog.go index bd0ada2f..dfbed965 100644 --- a/livelog/livelog.go +++ b/livelog/livelog.go @@ -52,27 +52,28 @@ type Writer struct { history []*logstream.Line prev []byte - closed bool - close chan struct{} - ready chan struct{} - - lastFlushTime time.Time + closed bool + close chan struct{} + ready chan struct{} + trimNewLineSuffix bool + lastFlushTime time.Time } // New returns a new writer -func New(client logstream.Client, key, name string, nudges []logstream.Nudge, printToStdout bool) *Writer { +func New(client logstream.Client, key, name string, nudges []logstream.Nudge, printToStdout bool, trimNewLineSuffix bool) *Writer { b := &Writer{ - client: client, - key: key, - name: name, - now: time.Now(), - printToStdout: printToStdout, - limit: defaultLimit, - interval: defaultInterval, - nudges: nudges, - close: make(chan struct{}), - ready: make(chan struct{}, 1), - lastFlushTime: time.Now(), + client: client, + key: key, + name: name, + now: time.Now(), + printToStdout: printToStdout, + limit: defaultLimit, + interval: defaultInterval, + nudges: nudges, + close: make(chan struct{}), + ready: make(chan struct{}, 1), + lastFlushTime: time.Now(), + trimNewLineSuffix: trimNewLineSuffix, } go b.Start() return b @@ -114,6 +115,11 @@ func (b *Writer) Write(p []byte) (n int, err error) { if part == "" { continue } + + if b.trimNewLineSuffix { + part = strings.TrimSuffix(part, "\n") + } + line := &logstream.Line{ Level: defaultLevel, Message: truncate(part, maxLineLimit), diff --git a/livelog/livelog_test.go b/livelog/livelog_test.go index 67435c38..7135cee4 100644 --- a/livelog/livelog_test.go +++ b/livelog/livelog_test.go @@ -16,7 +16,7 @@ import ( func TestLineWriterSingle(t *testing.T) { client := new(mockClient) - w := New(client, "1", "1", nil, false) + w := New(client, "1", "1", nil, false, false) w.SetInterval(time.Duration(0)) w.num = 4 _, _ = w.Write([]byte("foo\nbar\n")) @@ -40,6 +40,32 @@ func TestLineWriterSingle(t *testing.T) { } } +func TestLineWriterSingleWithTrimNewLineSuffixEnabled(t *testing.T) { + client := new(mockClient) + w := New(client, "1", "1", nil, false, true) + w.SetInterval(time.Duration(0)) + w.num = 4 + _, _ = w.Write([]byte("foo\nbar\n")) + + a := w.pending + b := []*logstream.Line{ + {Number: 4, Message: "foo"}, + {Number: 5, Message: "bar"}, + } + if err := compare(a, b); err != nil { + t.Fail() + fmt.Print(a) + t.Log(err) + } + + w.Close() + a = client.uploaded + if err := compare(a, b); err != nil { + t.Fail() + t.Log(err) + } +} + func compare(a, b []*logstream.Line) error { if len(a) != len(b) { return fmt.Errorf("expected size: %d, actual: %d", len(a), len(b)) diff --git a/pipeline/runtime/common.go b/pipeline/runtime/common.go index 30713a29..90b05eb1 100644 --- a/pipeline/runtime/common.go +++ b/pipeline/runtime/common.go @@ -191,7 +191,7 @@ func GetReplacer( cfg api.LogConfig, logKey, name string, secrets []string, ) logstream.Writer { client := getLogServiceClient(cfg) - wc := livelog.New(client, logKey, name, []logstream.Nudge{}, false) + wc := livelog.New(client, logKey, name, []logstream.Nudge{}, false, cfg.TrimNewLineSuffix) return logstream.NewReplacer(wc, secrets) } diff --git a/pipeline/runtime/step_executor.go b/pipeline/runtime/step_executor.go index f299d0aa..bcca935d 100644 --- a/pipeline/runtime/step_executor.go +++ b/pipeline/runtime/step_executor.go @@ -389,7 +389,8 @@ func getLogStreamWriter(r *api.StartStepRequest) logstream.Writer { // Create a log stream for step logs client := pipelineState.GetLogStreamClient() - wc := livelog.New(client, r.LogKey, r.Name, getNudges(), false) + + wc := livelog.New(client, r.LogKey, r.Name, getNudges(), false, pipelineState.GetLogConfig().TrimNewLineSuffix) wr := logstream.NewReplacer(wc, secrets) go wr.Open() //nolint:errcheck return wr diff --git a/pipeline/state.go b/pipeline/state.go index 7b372c5a..c21dce1f 100644 --- a/pipeline/state.go +++ b/pipeline/state.go @@ -81,6 +81,13 @@ func (s *State) GetTIConfig() *tiCfg.Cfg { return &s.tiConfig } +func (s *State) GetLogConfig() *api.LogConfig { + s.mu.Lock() + defer s.mu.Unlock() + + return &s.logConfig +} + func GetState() *State { once.Do(func() { state = &State{