Elenpay · markettes · Mar 27, 2024 · Mar 27, 2024 · Mar 27, 2024 · Mar 27, 2024
diff --git a/.env b/.env
@@ -1,12 +1,13 @@
 LNDCONNECTURIS=lndconnect://127.0.0.1:10003?cert=MIICJTCCAcygAwIBAgIQLYfp6m1vP9wFBXOcE-UsaDAKBggqhkjOPQQDAjAxMR8wHQYDVQQKExZsbmQgYXV0b2dlbmVyYXRlZCBjZXJ0MQ4wDAYDVQQDEwVjYXJvbDAeFw0yMzAzMjkxNTM4MjBaFw0yNDA1MjMxNTM4MjBaMDExHzAdBgNVBAoTFmxuZCBhdXRvZ2VuZXJhdGVkIGNlcnQxDjAMBgNVBAMTBWNhcm9sMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEcXT4dekJnAiZWd8Pk3FgL1BSFXMRwLGSAlk7Di5hIJnIA1B_o8RWKzlPz7u3Aw5mmWHhN8B2MWMylWlWB2130KOBxTCBwjAOBgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH_BAUwAwEB_zAdBgNVHQ4EFgQUDOS-19_0LFGf62WRyaaUSLc3j98wawYDVR0RBGQwYoIFY2Fyb2yCCWxvY2FsaG9zdIIFY2Fyb2yCDnBvbGFyLW4xLWNhcm9sggR1bml4ggp1bml4cGFja2V0ggdidWZjb25uhwR_AAABhxAAAAAAAAAAAAAAAAAAAAABhwSsFQAFMAoGCCqGSM49BAMCA0cAMEQCHxYe59PCXrTtSmGsOjfQo6V-sS8j73cqWOzTQbvgI3gCIQCj7sOxnZWBwilec7t8bBXjwPgX9frv8408JW4QhNFOUg&macaroon=AgEDbG5kAvgBAwoQHsW2NwwWb2yOKFMWQQkUWhIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgjpV-eOw554EPrSXPxDhQuOnnwHmEO47Hu1Uiu6EiMNY
 POLLINGINTERVAL=5s
 RETRIESBEFOREBACKOFF=3
-BACKOFFCOEFFICIENT=0.99
-BACKOFFLIMIT=0.1
+BACKOFFCOEFFICIENT=0.90
+BACKOFFLIMIT=0.4
 LOGLEVEL=DEBUG
 LOGFORMAT=text
 NODEGUARDHOST=localhost:50051
 NODEGUARD_API_KEY=8rvSsUGeyXXdDQrHctcTey/xtHdZQEn945KHwccKp9Q=
 LOOPDCONNECTURIS=lndconnect://localhost:11010?cert=MIIC1zCCAn2gAwIBAgIQRWr7OVVAidkjx3rY1Fui1DAKBggqhkjOPQQDAjA6MSAwHgYDVQQKExdsb29wIGF1dG9nZW5lcmF0ZWQgY2VydDEWMBQGA1UEAxMNRG9vbW1hYy5sb2NhbDAeFw0yMzA0MTExMDIyMTJaFw0yNDA2MDUxMDIyMTJaMDoxIDAeBgNVBAoTF2xvb3AgYXV0b2dlbmVyYXRlZCBjZXJ0MRYwFAYDVQQDEw1Eb29tbWFjLmxvY2FsMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEZ4oLXzeFkXnnnjZvbHoJLWTBkzovapwkv43lMFAGvD3EUYjapn9UN63LOamwSW67S2p4fyDbZykRVi5KdGuHWqOCAWMwggFfMA4GA1UdDwEB_wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH_MB0GA1UdDgQWBBQ_C-VsxCCp7hIukcSQY66T_1-wwDCCAQYGA1UdEQSB_jCB-4INRG9vbW1hYy5sb2NhbIIJbG9jYWxob3N0ggR1bml4ggp1bml4cGFja2V0ggdidWZjb25uhwR_AAABhxAAAAAAAAAAAAAAAAAAAAABhxD-gAAAAAAAAAAAAAAAAAABhxD-gAAAAAAAAFSCWv_--hJchxD-gAAAAAAAAFSCWv_--hJahxD-gAAAAAAAAFSCWv_--hJbhxD-gAAAAAAAAAAtb1hMXkTshwTAqAHlhxD-gAAAAAAAAHQf-P_-JUMfhwQKkK07hxD-gAAAAAAAANx-KvMiB6mphxD-gAAAAAAAAIdaqQYVUHbchxD-gAAAAAAAAM6BCxy9LAaeMAoGCCqGSM49BAMCA0gAMEUCIHmOfHbewxWmX0JoItg8vF84A3GWfYLvjbnGxoapsaH4AiEAlGk3QVSDSHjP7vntLEK37tJ3ZCZAYeuLJKYSjbgbdI8&macaroon=AgEEbG9vcAJ3AwoQSQHLuEtbfDxGeNmwvABdRRIBMBoMCgRhdXRoEgRyZWFkGg8KBGxvb3ASAmluEgNvdXQaGgoLc3VnZ2VzdGlvbnMSBHJlYWQSBXdyaXRlGhUKBHN3YXASB2V4ZWN1dGUSBHJlYWQaDQoFdGVybXMSBHJlYWQAAAYgHb_8HcZwROzXASOxItpkbD3XAWHy1Ye32vlc1ccLeg8
 OTEL_RESOURCE_ATTRIBUTES=service.name=liquidator,service.version=0.1.0
-OTEL_EXPORTER_OTLP_ENDPOINT=host.docker.internal:4317
+OTEL_EXPORTER_OTLP_ENDPOINT=host.docker.internal:4317
+SLEEPMAX=500ms
diff --git a/README.md b/README.md
@@ -1,11 +1,15 @@
 # NodeGuard Liquidator
+
 [![Go Report Card](https://goreportcard.com/badge/github.com/Elenpay/liquidator)](https://goreportcard.com/report/github.com/Elenpay/liquidator)
+
 <p align="center">
   <img src="liquidator.png" width="100px" />
 </p>
 
 # Description
+
 A CLI tool to monitor and automate the liquidity of your LND channels
+
 ```
 Usage:
   liquidator [flags]
@@ -25,8 +29,11 @@ Flags:
       --swapPublicationOffset string   Swap publication deadline offset (Maximum time for the swap provider to publish the swap) (default "60m")
       --sweepConfTarget string         Target number of confirmations for swaps, this uses bitcoin core broken estimator, procced with caution (default "400")
 ```
+
 # Requirements
+
 This project uses [just](https://github.com/casey/just) with the following recipes
+
 ```
 Available recipes:
     build
@@ -50,6 +57,7 @@ Available recipes:
     test
     unzip-loopd-datadir
 ```
+
 # Environment Variables / Flags
 
 All the flags can be set as environment variables, with the following format, except stated, they are all mandatory:
@@ -64,53 +72,71 @@ All the flags can be set as environment variables, with the following format, ex
 - RETRIESBEFOREBACKOFF (optional) : Number of retries before applying backoff to the swap (default: 3)
 - BACKOFFCOEFFICIENT (optional) : Coefficient to apply to the backoff (default: 0.95)
 - BACKOFFLIMIT (optional) : Limit coefficient of the backoff (default: 0.1)
+- SLEEPBETWEENRETRIES (optional) : Base time to sleep between retries (default: 500ms)
+- SLEEPBETWEENRETRIESBACKOFF (optional) : Coefficient to apply to the backoff (default: 1.5)
+- SLEEPMAX (optional) : Maximum time to sleep between retries (default: 60s)
 
 # Build & test
+
 ## Build
+
 ```
 just build
 ```
 
 ## Testing
+
 ```
 just test
 ```
+
 # Dev environment
+
 1. Launch a local regtest with polar from regtest.polar.zip
 2. Lauch with VS Code pre-defined configuration
 
 ## Setup loop/loopd/loopserver regtest environment
+
 1. Make sure your regtest.polar.zip network in polar is running
 2. Unzip loopd datadir
-````
+
+```
 just unzip-loopd-datadir
-````
+```
+
 3. Get git submodules
-````
+
+```
 just init-submodules
-````
+```
+
 4. Compile and install loopd/loop binaries (Make sure your golang install bin dir is reachable from your PATH)
-````
+
+```
 just install-loopd-loop
-````
+```
+
 5. Using just, run the following command:
-````
+
+```
 just start-all
-````
+```
+
 6. This comand should have build a `loopserver `docker image, and started a `loopserver` container along with loopd as a native binary.
 
 ## Loop just recipes
-There are a few recipes using `just -l` to interact with loopd for loop in, loop out and calling loop CLI with args (`just loop <args>`).
-
-
 
+There are a few recipes using `just -l` to interact with loopd for loop in, loop out and calling loop CLI with args (`just loop <args>`).
 
 # Metrics
+
 The following metrics are exposed in the `/metrics` endpoint on port `9000` (e.g. `localhost:9000/metrics`)):
+
 - `liquidator_channel_balance`: Channel balance ratio in 0 to 1 range, 0 means all the balance is on the local side of the channel, 1 means all the balance is on the remote side of the channel
- 
+
 Example:
- ```
+
+```
 liquidator_channel_balance{active="false",chan_id="118747255865345",local_node_alias="alice",local_node_pubkey="03b48034270e522e4033afdbe43383d66d426638927b940d09a8a7a0de4d96e807",remote_node_alias="",remote_node_pubkey="02f97d034c6c8f5ad95b1fe6abfe68ab154e85b1f5bb909815baeb5c8a46cdf622",initiator="false"} 0.99
 
 liquidator_channel_balance{active="false",chan_id="125344325632000",local_node_alias="alice",local_node_pubkey="03b48034270e522e4033afdbe43383d66d426638927b940d09a8a7a0de4d96e807",remote_node_alias="",remote_node_pubkey="02f97d034c6c8f5ad95b1fe6abfe68ab154e85b1f5bb909815baeb5c8a46cdf622",initiator="false"} 0

diff --git a/flags.go b/flags.go
@@ -24,6 +24,7 @@ package main
 import (
 	"os"
 	"strings"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 
@@ -121,6 +122,18 @@ func init() {
 	rootCmd.Flags().String("sweepConfTarget", "400", "Target number of confirmations for swaps, this uses bitcoin core broken estimator, procced with caution")
 	viper.BindPFlag("sweepConfTarget", rootCmd.Flags().Lookup("sweepConfTarget"))
 
+	//Sleep between retries
+	rootCmd.Flags().Duration("sleepBetweenRetries", 500*time.Millisecond, "Sleep between retries")
+	viper.BindPFlag("sleepBetweenRetries", rootCmd.Flags().Lookup("sleepBetweenRetries"))
+
+	//Sleep between retries backoff
+	rootCmd.Flags().Float64("sleepBetweenRetriesBackoff", 1.5, "Sleep between retries backoff")
+	viper.BindPFlag("sleepBetweenRetriesBackoff", rootCmd.Flags().Lookup("sleepBetweenRetriesBackoff"))
+
+	//Sleep max
+	rootCmd.Flags().Duration("sleepMax", 60*time.Second, "Sleep max")
+	viper.BindPFlag("sleepMax", rootCmd.Flags().Lookup("sleepMax"))
+
 	//Now we set the global vars
 
 	pollingInterval = viper.GetDuration("pollingInterval")
@@ -133,6 +146,10 @@ func init() {
 	backoffLimit = viper.GetFloat64("backoffLimit")
 	limitFees = viper.GetFloat64("limitFees")
 
+	sleepBetweenRetries = viper.GetDuration("sleepBetweenRetries")
+	sleepBetweenRetriesBackoff = viper.GetFloat64("sleepBetweenRetriesBackoff")
+	sleepMax = viper.GetDuration("sleepMax")
+
 	//Set log level and format
 
 	logLevel, err := log.ParseLevel(viper.GetString("logLevel"))
@@ -158,5 +175,9 @@ func init() {
 	log.Debug("nodeguardHost: ", nodeguardHost)
 	log.Debug("retriesBeforeBackoff: ", viper.GetInt("retriesBeforeBackoff"))
 	log.Debug("backoffCoefficient: ", viper.GetFloat64("backoffCoefficient"))
-
+	log.Debug("backoffLimit: ", viper.GetFloat64("backoffLimit"))
+	log.Debug("limitFees: ", viper.GetFloat64("limitFees"))
+	log.Debug("sleepBetweenRetries: ", viper.GetDuration("sleepBetweenRetries"))
+	log.Debug("sleepBetweenRetriesBackoff: ", viper.GetFloat64("sleepBetweenRetriesBackoff"))
+	log.Debug("sleepMax: ", viper.GetDuration("sleepMax"))
 }
diff --git a/liquidator.go b/liquidator.go
@@ -30,12 +30,15 @@ import (
 )
 
 var (
-	prometheusMetrics  *metrics
-	rulesCache         cache.Cache
-	retries            int
-	backoffCoefficient float64
-	backoffLimit       float64
-	limitFees          float64
+	prometheusMetrics          *metrics
+	rulesCache                 cache.Cache
+	retries                    int
+	backoffCoefficient         float64
+	backoffLimit               float64
+	limitFees                  float64
+	sleepBetweenRetries        time.Duration
+	sleepBetweenRetriesBackoff float64
+	sleepMax                   time.Duration
 )
 
 // Entrypoint of liquidator main logic
@@ -475,7 +478,7 @@ func manageChannelLiquidity(info ManageChannelLiquidityInfo) error {
 
 			retryCounter := 1
 
-			err := performReverseSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget)
+			err := performReverseSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget, sleepBetweenRetries)
 			if err != nil {
 				return err
 			}
@@ -495,7 +498,7 @@ func manageChannelLiquidity(info ManageChannelLiquidityInfo) error {
 
 			retryCounter := 1
 
-			err := performSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget)
+			err := performSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget, sleepBetweenRetries)
 			if err != nil {
 				return err
 			}
@@ -507,7 +510,7 @@ func manageChannelLiquidity(info ManageChannelLiquidityInfo) error {
 
 }
 
-func performSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel, swapAmount int64, rule nodeguard.LiquidityRule, span trace.Span, loopdCtx context.Context, retryCounter int, swapAmountTarget int64) error {
+func performSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel, swapAmount int64, rule nodeguard.LiquidityRule, span trace.Span, loopdCtx context.Context, retryCounter int, swapAmountTarget int64, sleepBetweenRetries time.Duration) error {
 	//Perform the swap
 	swapRequest := provider.SubmarineSwapRequest{
 		SatsAmount:    swapAmount,
@@ -606,15 +609,22 @@ func performSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel, swapAm
 			//Retry the swap
 			retryCounter++
 			log.WithField("span", span).Infof("retrying swap for channel %v, retry number: %v/%v on node %v", channel.GetChanId(), retryCounter, retries, info.nodeInfo.GetIdentityPubkey())
-			err := performSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget)
+			time.Sleep(sleepBetweenRetries)
+			err := performSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget, sleepBetweenRetries)
 			if err != nil {
 				return err
 			}
 		} else {
 			limitSwapAmount := float64(helper.AbsInt64((channel.RemoteBalance - swapAmountTarget))) * backoffLimit
 			if limitSwapAmount < float64(swapAmount)*backoffCoefficient {
+				time.Sleep(sleepBetweenRetries)
 				newSwapAmount := int64(float64(swapAmount) * backoffCoefficient)
-				err := performSwap(info, channel, newSwapAmount, rule, span, loopdCtx, 0, swapAmountTarget)
+				newSecondsAmount := sleepBetweenRetries.Seconds() * sleepBetweenRetriesBackoff
+				newSleepBetweenRetries := time.Duration(newSecondsAmount) * time.Second
+				if newSleepBetweenRetries > sleepMax {
+					newSleepBetweenRetries = sleepMax
+				}
+				err := performSwap(info, channel, newSwapAmount, rule, span, loopdCtx, 0, swapAmountTarget, newSleepBetweenRetries)
 				if err != nil {
 					return err
 				}
@@ -636,7 +646,7 @@ func performSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel, swapAm
 	return nil
 }
 
-func performReverseSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel, swapAmount int64, rule nodeguard.LiquidityRule, span trace.Span, loopdCtx context.Context, retryCounter int, swapAmountTarget int64) error {
+func performReverseSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel, swapAmount int64, rule nodeguard.LiquidityRule, span trace.Span, loopdCtx context.Context, retryCounter int, swapAmountTarget int64, sleepBetweenRetries time.Duration) error {
 	// Check if it is a reverse swap to a wallet or to an address
 	var address string
 	if rule.IsReverseSwapWalletRule {
@@ -672,7 +682,8 @@ func performReverseSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel,
 			limitSwapAmount := float64(helper.AbsInt64((channel.RemoteBalance - swapAmountTarget))) * backoffLimit
 			if limitSwapAmount < float64(swapAmount)*backoffCoefficient {
 				newSwapAmount := int64(float64(swapAmount) * backoffCoefficient)
-				err = performReverseSwap(info, channel, newSwapAmount, rule, span, loopdCtx, 0, swapAmountTarget)
+				time.Sleep(sleepBetweenRetries)
+				err = performReverseSwap(info, channel, newSwapAmount, rule, span, loopdCtx, 0, swapAmountTarget, sleepBetweenRetries)
 				if err != nil {
 					return err
 				}
@@ -698,15 +709,22 @@ func performReverseSwap(info ManageChannelLiquidityInfo, channel *lnrpc.Channel,
 			//Retry the swap
 			retryCounter++
 			log.WithField("span", span).Infof("retrying reverse swap for channel %v, retry number: %v/%v on node %v", channel.GetChanId(), retryCounter, retries, info.nodeInfo.GetIdentityPubkey())
-			err := performReverseSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget)
+			time.Sleep(sleepBetweenRetries)
+			err := performReverseSwap(info, channel, swapAmount, rule, span, loopdCtx, retryCounter, swapAmountTarget, sleepBetweenRetries)
 			if err != nil {
 				return err
 			}
 		} else {
 			limitSwapAmount := float64(helper.AbsInt64((channel.RemoteBalance - swapAmountTarget))) * backoffLimit
 			if limitSwapAmount < float64(swapAmount)*backoffCoefficient {
+				time.Sleep(sleepBetweenRetries)
 				newSwapAmount := int64(float64(swapAmount) * backoffCoefficient)
-				err := performReverseSwap(info, channel, newSwapAmount, rule, span, loopdCtx, 0, swapAmountTarget)
+				newMsAmount := float64((sleepBetweenRetries.Milliseconds())) * sleepBetweenRetriesBackoff
+				newSleepBetweenRetries := time.Duration(newMsAmount) * time.Millisecond
+				if newSleepBetweenRetries > sleepMax {
+					newSleepBetweenRetries = sleepMax
+				}
+				err := performReverseSwap(info, channel, newSwapAmount, rule, span, loopdCtx, 0, swapAmountTarget, newSleepBetweenRetries)
 				if err != nil {
 					return err
 				}

diff --git a/liquidator_test.go b/liquidator_test.go
@@ -505,5 +505,5 @@ func Test_monitorChannel(t *testing.T) {
 	}
 
 	// Wait for the goroutine to finish
-	time.Sleep(1 * time.Second)
+	time.Sleep(30 * time.Second)
 }