Merge pull request #1126 from lavanet/CNS-769-make-payment-e-2-e-stable

CNS-769: make payment e2e stable

.github/workflows/e2e.yml

@@ -197,49 +197,49 @@ jobs:
         if: always()
         continue-on-error: true
         run: grep "" testutil/e2e/sdkLogs/01_sdkTest* --include="*errors*"
-  # test-payment-e2e:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #       with:
-  #         fetch-depth: "0"
-  #     - uses: actions/setup-go@v3
-  #       with:
-  #         go-version: 1.20.5
+  test-payment-e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: "0"
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.5
 
       ######################################################
-      ### Run Lava Protocol Payment E2E Tests - temporarily disabled
+      ### Run Lava Protocol Payment E2E Tests
       ######################################################
-      # - name: Run Lava Payment E2E Tests -timeout 1200s
-      #   run: go test ./testutil/e2e/ -run ^TestLavaProtocolPayment$ -v -timeout 1200s # 20mins
-
-      # - name: tail -n 300 Lavad Logs
-      #   if: always()
-      #   run: tail -n 300 testutil/e2e/protocolLogs/00_StartLava.log
-
-      # - name: Stake Lava All Logs
-      #   if: always()
-      #   run: cat testutil/e2e/protocolLogs/01_stakeLavaForPayment.log
-
-      # - name: Stake Lava Error Only Logs
-      #   if: always()
-      #   continue-on-error: true
-      #   run: cat testutil/e2e/protocolLogs/01_stakeLava_errors.log
-
-      # - name: Lava Provider All Logs
-      #   if: always()
-      #   run: grep "" testutil/e2e/protocolLogs/05_LavaProvider* --exclude="*errors*"
-
-      # - name: Lava Provider Error Only Logs
-      #   if: always()
-      #   continue-on-error: true
-      #   run: grep "" testutil/e2e/protocolLogs/05_LavaProvider* --include="*errors*"
-
-      # - name: RPCConsumer Consumer All Logs
-      #   if: always()
-      #   run: grep "" testutil/e2e/protocolLogs/06_RPCConsumer* --exclude="*errors*"
-
-      # - name: RPCConsumer Consumer Error Only Logs
-      #   if: always()
-      #   continue-on-error: true
-      #   run: grep "" testutil/e2e/protocolLogs/06_RPCConsumer* --include="*errors*"
+      - name: Run Lava Payment E2E Tests -timeout 1200s
+        run: go test ./testutil/e2e/ -run ^TestLavaProtocolPayment$ -v -timeout 1200s # 20mins
+
+      - name: tail -n 300 Lavad Logs
+        if: always()
+        run: tail -n 300 testutil/e2e/protocolLogs/00_StartLava.log
+
+      - name: Stake Lava All Logs
+        if: always()
+        run: cat testutil/e2e/protocolLogs/01_stakeLavaForPayment.log
+
+      - name: Stake Lava Error Only Logs
+        if: always()
+        continue-on-error: true
+        run: cat testutil/e2e/protocolLogs/01_stakeLava_errors.log
+
+      - name: Lava Provider All Logs
+        if: always()
+        run: grep "" testutil/e2e/protocolLogs/05_LavaProvider* --exclude="*errors*"
+
+      - name: Lava Provider Error Only Logs
+        if: always()
+        continue-on-error: true
+        run: grep "" testutil/e2e/protocolLogs/05_LavaProvider* --include="*errors*"
+
+      - name: RPCConsumer Consumer All Logs
+        if: always()
+        run: grep "" testutil/e2e/protocolLogs/06_RPCConsumer* --exclude="*errors*"
+
+      - name: RPCConsumer Consumer Error Only Logs
+        if: always()
+        continue-on-error: true
+        run: grep "" testutil/e2e/protocolLogs/06_RPCConsumer* --include="*errors*"

scripts/init_chain.sh

@@ -38,6 +38,8 @@ if [ "$1" == "debug" ]; then
         | jq '.app_state.crisis.constant_fee.denom = "ulava"' \
         | jq '.app_state.epochstorage.params.epochsToSave = "5"' \
         | jq '.app_state.epochstorage.params.epochBlocks = "4"' \
+        | jq '.app_state.distribution.params.community_tax = "0"' \
+        | jq '.app_state.rewards.params.validators_subscription_participation = "0"' \
         | jq '.app_state.downtime.params.downtime_duration = "1s"' \
     )
 else
@@ -105,8 +107,12 @@ done
 
 # add validators_allocation_pool for validators block rewards
 # its total balance is 3% from the total tokens amount: 10^9 * 10^6 ulava
-lavad add-genesis-account validators_rewards_allocation_pool 30000000000000ulava --module-account 
-lavad add-genesis-account providers_rewards_allocation_pool 30000000000000ulava --module-account 
+lavad add-genesis-account validators_rewards_allocation_pool 30000000000000ulava --module-account
+if [ "$1" == "debug" ]; then
+    lavad add-genesis-account providers_rewards_allocation_pool 0ulava --module-account
+else
+    lavad add-genesis-account providers_rewards_allocation_pool 30000000000000ulava --module-account 
+fi
 lavad gentx alice 10000000000000ulava --chain-id $chainID
 lavad collect-gentxs
 lavad start --pruning=nothing

testutil/e2e/allowedErrorList.go

@@ -14,3 +14,8 @@ var allowedErrors = map[string]string{
 var allowedErrorsDuringEmergencyMode = map[string]string{
 	"connection refused": "Connection to tendermint port sometimes can happen as we shut down the node and we try to fetch info during emergency mode",
 }
+
+var allowedErrorsPaymentE2E = map[string]string{
+	"conflict with is already open for this client": "This error is allowed because it's unrelated to payment E2E",
+	"could not get pairing":                         "This error is allowed because the test passes and then there's a random instance of this error. It's ok to allow it because if there was no pairing in a critical step of the test, the test would fail since it checks payments",
+}

testutil/e2e/paymentE2E.go

@@ -1,13 +1,16 @@
 package e2e
 
 import (
+	"bufio"
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"go/build"
 	"math"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -170,6 +173,88 @@ func (lt *lavaTest) checkPayment(providers []string, startRewards []sdk.Coin) {
 	}
 }
 
+func (lt *lavaTest) saveLogsForPayment() {
+	if _, err := os.Stat(lt.logPath); errors.Is(err, os.ErrNotExist) {
+		err = os.MkdirAll(lt.logPath, os.ModePerm)
+		if err != nil {
+			panic(err)
+		}
+	}
+	errorFound := false
+	errorFiles := []string{}
+	errorPrint := make(map[string]string)
+	for fileName, logBuffer := range lt.logs {
+		file, err := os.Create(lt.logPath + fileName + ".log")
+		if err != nil {
+			panic(err)
+		}
+		writer := bufio.NewWriter(file)
+		writer.Write(logBuffer.Bytes())
+		writer.Flush()
+		utils.LavaFormatDebug("writing file", []utils.Attribute{{Key: "fileName", Value: fileName}, {Key: "lines", Value: len(logBuffer.Bytes())}}...)
+		file.Close()
+
+		lines := strings.Split(logBuffer.String(), "\n")
+		errorLines := []string{}
+		for _, line := range lines {
+			if fileName == "00_StartLava" { // TODO remove this and solve the errors
+				break
+			}
+			if strings.Contains(line, " ERR ") || strings.Contains(line, "[Error]" /* sdk errors*/) {
+				isAllowedError := false
+				for errorSubstring := range allowedErrors {
+					if strings.Contains(line, errorSubstring) {
+						isAllowedError = true
+						break
+					}
+				}
+
+				for errorSubstring := range allowedErrorsPaymentE2E {
+					if strings.Contains(line, errorSubstring) {
+						isAllowedError = true
+						break
+					}
+				}
+
+				// When test did not finish properly save all logs. If test finished properly save only non allowed errors.
+				if !lt.testFinishedProperly || !isAllowedError {
+					errorFound = true
+					errorLines = append(errorLines, line)
+				}
+			}
+		}
+		if len(errorLines) == 0 {
+			continue
+		}
+
+		// dump all errors into the log file
+		errors := strings.Join(errorLines, "\n")
+		errFile, err := os.Create(lt.logPath + fileName + "_errors.log")
+		if err != nil {
+			panic(err)
+		}
+		writer = bufio.NewWriter(errFile)
+		writer.Write([]byte(errors))
+		writer.Flush()
+		errFile.Close()
+
+		// keep at most 5 errors to display
+		count := len(errorLines)
+		if count > 5 {
+			count = 5
+		}
+		errorPrint[fileName] = strings.Join(errorLines[:count], "\n")
+		errorFiles = append(errorFiles, fileName)
+	}
+
+	if errorFound {
+		for _, errLine := range errorPrint {
+			fmt.Println("ERROR: ", errLine)
+		}
+		panic("Error found in logs on " + lt.logPath + strings.Join(errorFiles, ", "))
+	}
+}
+
 func withinRange(value1, value2, percentage uint64) bool {
 	maxDifference := value1 * percentage / 100
 	return math.Abs(float64(value1)-float64(value2)) <= float64(maxDifference)
@@ -207,13 +292,13 @@ func runPaymentE2E(timeout time.Duration) {
 	// use defer to save logs in case the tests fail
 	defer func() {
 		if r := recover(); r != nil {
-			lt.saveLogs()
+			lt.saveLogsForPayment()
 			for _, cmd := range lt.commands {
 				cmd.Process.Kill()
 			}
 			panic("E2E Failed")
 		} else {
-			lt.saveLogs()
+			lt.saveLogsForPayment()
 		}
 	}()