sqlite: string compare database is locked error message

persist/sqlite/consts_default.go

@@ -2,10 +2,13 @@
 
 package sqlite
 
+import "time"
+
 const (
-	busyTimeout   = 5000 // 5 seconds
-	retryAttempts = 15   // 15 attempts
-	factor        = 2.0  // factor ^ retryAttempts = backoff time in milliseconds
+	busyTimeout      = 10000 // 10 seconds
+	maxRetryAttempts = 30    // 30 attempts
+	factor           = 1.8   // factor ^ retryAttempts = backoff time in milliseconds
+	maxBackoff       = 15 * time.Second
 
 	// the number of records to limit long-running sector queries to
 	sqlSectorBatchSize = 256 // 1 GiB

persist/sqlite/consts_testing.go

@@ -2,10 +2,13 @@
 
 package sqlite
 
+import "time"
+
 const (
-	busyTimeout   = 5   // 5ms
-	retryAttempts = 10  // 10 attempts
-	factor        = 2.0 // factor ^ retryAttempts = backoff time in milliseconds
+	busyTimeout      = 100 // 100ms
+	maxRetryAttempts = 10  // 10 attempts
+	factor           = 2.0 // factor ^ retryAttempts = backoff time in milliseconds
+	maxBackoff       = 15 * time.Second
 
 	// the number of records to limit long-running sector queries to
 	sqlSectorBatchSize = 5 // 20 MiB

persist/sqlite/store.go

@@ -79,14 +79,14 @@ func (s *Store) queryRow(query string, args ...any) *loggedRow {
 // transaction executes a function within a database transaction. If the
 // function returns an error, the transaction is rolled back. Otherwise, the
 // transaction is committed. If the transaction fails due to a busy error, it is
-// retried up to 10 times before returning.
+// retried up to 15 times before returning.
 func (s *Store) transaction(fn func(txn) error) error {
 	var err error
 	txnID := hex.EncodeToString(frand.Bytes(4))
 	log := s.log.Named("transaction").With(zap.String("id", txnID))
 	start := time.Now()
 	attempt := 1
-	for ; attempt <= retryAttempts; attempt++ {
+	for ; attempt < maxRetryAttempts; attempt++ {
 		attemptStart := time.Now()
 		log := log.With(zap.Int("attempt", attempt))
 		err = doTransaction(s.db, log, fn)
@@ -99,8 +99,13 @@ func (s *Store) transaction(fn func(txn) error) error {
 		if !strings.Contains(err.Error(), "database is locked") {
 			break
 		}
-		log.Debug("database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"))
-		jitterSleep(time.Duration(math.Pow(factor, float64(attempt))) * time.Millisecond) // exponential backoff
+		// exponential backoff
+		sleep := time.Duration(math.Pow(factor, float64(attempt))) * time.Millisecond
+		if sleep > maxBackoff {
+			sleep = maxBackoff
+		}
+		log.Debug("database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"), zap.Duration("retry", sleep))
+		jitterSleep(sleep)
 	}
 	return fmt.Errorf("transaction failed (%d): %w", attempt, err)
 }