From a9890990879f487e6d47d3a166114fccf8c3da0b Mon Sep 17 00:00:00 2001
From: or-shachar <or.shachar@wiz.io>
Date: Wed, 22 Nov 2023 20:49:36 +0200
Subject: [PATCH] Tolerate remote put errors

---
 cachers/combined.go | 9 ++++-----
 cachers/s3.go       | 5 +++++
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/cachers/combined.go b/cachers/combined.go
index 8286369..160bfbd 100644
--- a/cachers/combined.go
+++ b/cachers/combined.go
@@ -91,7 +91,8 @@ func (l *CombinedCache) Get(ctx context.Context, actionID string) (outputID, dis
 	}
 	return outputID, diskPath, nil
 }
-func (l *CombinedCache) Put(ctx context.Context, actionID, outputID string, size int64, body io.Reader) (diskPath string, err error) {
+
+func (l *CombinedCache) Put(ctx context.Context, actionID, outputID string, size int64, body io.Reader) (string, error) {
 	pr, pw := io.Pipe()
 	diskPutCh := make(chan any, 1)
 	go func() {
@@ -117,14 +118,12 @@ func (l *CombinedCache) Put(ctx context.Context, actionID, outputID string, size
 
 		putBody = io.TeeReader(body, pw)
 	}
-	_, err = l.putsMetrics.DoWithMeasure(size, func() (string, error) {
+	// tolerate remote write errors
+	_, _ = l.putsMetrics.DoWithMeasure(size, func() (string, error) {
 		e := l.remoteCache.Put(ctx, actionID, outputID, size, putBody)
 		return "", e
 	})
 	pw.Close()
-	if err != nil {
-		return "", err
-	}
 	v := <-diskPutCh
 	if err, ok := v.(error); ok {
 		log.Printf("HTTPCache.Put local disk error: %v", err)
diff --git a/cachers/s3.go b/cachers/s3.go
index 5b78d30..90379c3 100644
--- a/cachers/s3.go
+++ b/cachers/s3.go
@@ -82,7 +82,12 @@ func (s *S3Cache) Put(ctx context.Context, actionID, outputID string, size int64
 		Metadata: map[string]string{
 			outputIDMetadataKey: outputID,
 		},
+	}, func(options *s3.Options) {
+		options.RetryMaxAttempts = 1 // We cannot perform seek in Body
 	})
+	if err != nil && s.verbose {
+		log.Printf("error S3 put for %s:  %v", actionKey, err)
+	}
 	return
 }