fix(consensus): stopped abci client is not recoverable

abci/client/socket_client.go

@@ -38,6 +38,10 @@ type socketClient struct {
 
 var _ Client = (*socketClient)(nil)
 
+var (
+	ErrClientStopped = errors.New("client has stopped")
+)
+
 // NewSocketClient creates a new socket client, which connects to a given
 // address. If mustConnect is true, the client will return an error upon start
 // if it fails to connect.
@@ -234,7 +238,7 @@ func (cli *socketClient) didRecvResponse(res *types.Response) error {
 
 func (cli *socketClient) doRequest(ctx context.Context, req *types.Request) (*types.Response, error) {
 	if !cli.IsRunning() {
-		return nil, errors.New("client has stopped")
+		return nil, ErrClientStopped
 	}
 
 	reqres := makeReqRes(ctx, req)

internal/consensus/state_try_add_commit.go

@@ -2,8 +2,10 @@ package consensus
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	abciclient "github.com/dashpay/tenderdash/abci/client"
 	"github.com/dashpay/tenderdash/dash"
 	cstypes "github.com/dashpay/tenderdash/internal/consensus/types"
 	"github.com/dashpay/tenderdash/libs/log"
@@ -108,6 +110,10 @@ func (cs *TryAddCommitAction) verifyCommit(ctx context.Context, stateData *State
 	// We have a correct block, let's process it before applying the commit
 	err = cs.blockExec.ensureProcess(ctx, &stateData.RoundState, commit.Round)
 	if err != nil {
+		if errors.Is(err, abciclient.ErrClientStopped) {
+			// this is a non-recoverable error in current architecture
+			panic(fmt.Errorf("ABCI client stopped, Tenderdash needs to be restarted: %w", err))
+		}
 		return false, fmt.Errorf("unable to process proposal: %w", err)
 	}
 	err = cs.blockExec.validate(ctx, stateData)