Tolerate remote put errors

or-shachar · Nov 22, 2023 · a989099 · a989099
1 parent f09b126
commit a989099
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/cachers/combined.go b/cachers/combined.go
@@ -91,7 +91,8 @@ func (l *CombinedCache) Get(ctx context.Context, actionID string) (outputID, dis
 	}
 	return outputID, diskPath, nil
 }
-func (l *CombinedCache) Put(ctx context.Context, actionID, outputID string, size int64, body io.Reader) (diskPath string, err error) {
+
+func (l *CombinedCache) Put(ctx context.Context, actionID, outputID string, size int64, body io.Reader) (string, error) {
 	pr, pw := io.Pipe()
 	diskPutCh := make(chan any, 1)
 	go func() {
@@ -117,14 +118,12 @@ func (l *CombinedCache) Put(ctx context.Context, actionID, outputID string, size
 
 		putBody = io.TeeReader(body, pw)
 	}
-	_, err = l.putsMetrics.DoWithMeasure(size, func() (string, error) {
+	// tolerate remote write errors
+	_, _ = l.putsMetrics.DoWithMeasure(size, func() (string, error) {
 		e := l.remoteCache.Put(ctx, actionID, outputID, size, putBody)
 		return "", e
 	})
 	pw.Close()
-	if err != nil {
-		return "", err
-	}
 	v := <-diskPutCh
 	if err, ok := v.(error); ok {
 		log.Printf("HTTPCache.Put local disk error: %v", err)

diff --git a/cachers/s3.go b/cachers/s3.go
@@ -82,7 +82,12 @@ func (s *S3Cache) Put(ctx context.Context, actionID, outputID string, size int64
 		Metadata: map[string]string{
 			outputIDMetadataKey: outputID,
 		},
+	}, func(options *s3.Options) {
+		options.RetryMaxAttempts = 1 // We cannot perform seek in Body
 	})
+	if err != nil && s.verbose {
+		log.Printf("error S3 put for %s:  %v", actionKey, err)
+	}
 	return
 }