feat: PRT - add offline spec feat

protocol/chainlib/common_test_utils.go

@@ -26,6 +26,7 @@ import (
 	"github.com/lavanet/lava/v2/protocol/lavasession"
 	testcommon "github.com/lavanet/lava/v2/testutil/common"
 	keepertest "github.com/lavanet/lava/v2/testutil/keeper"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	plantypes "github.com/lavanet/lava/v2/x/plans/types"
 	spectypes "github.com/lavanet/lava/v2/x/spec/types"
 	"github.com/stretchr/testify/require"
@@ -126,7 +127,7 @@ func CreateChainLibMocks(
 ) (cpar ChainParser, crout ChainRouter, cfetc chaintracker.ChainFetcher, closeServer func(), endpointRet *lavasession.RPCProviderEndpoint, errRet error) {
 	utils.SetGlobalLoggingLevel("debug")
 	closeServer = nil
-	spec, err := keepertest.GetASpec(specIndex, getToTopMostPath, nil, nil)
+	spec, err := specutils.GetASpec(specIndex, getToTopMostPath, nil, nil)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
@@ -250,7 +251,7 @@ func SetupForTests(t *testing.T, numOfProviders int, specID string, getToTopMost
 		ts.Providers = append(ts.Providers, testcommon.CreateNewAccount(ts.Ctx, *ts.Keepers, balance))
 	}
 	sdkContext := sdk.UnwrapSDKContext(ts.Ctx)
-	spec, err := keepertest.GetASpec(specID, getToTopMostPath, &sdkContext, &ts.Keepers.Spec)
+	spec, err := specutils.GetASpec(specID, getToTopMostPath, &sdkContext, &ts.Keepers.Spec)
 	if err != nil {
 		require.NoError(t, err)
 	}

protocol/chainlib/jsonRPC_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/lavanet/lava/v2/protocol/chainlib/chainproxy/rpcInterfaceMessages"
 	"github.com/lavanet/lava/v2/protocol/chainlib/extensionslib"
 	"github.com/lavanet/lava/v2/protocol/common"
-	keepertest "github.com/lavanet/lava/v2/testutil/keeper"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	plantypes "github.com/lavanet/lava/v2/x/plans/types"
 	spectypes "github.com/lavanet/lava/v2/x/spec/types"
 	"github.com/stretchr/testify/assert"
@@ -253,7 +253,7 @@ func TestExtensions(t *testing.T) {
 	configuredExtensions := map[string]struct{}{
 		"archive": {},
 	}
-	spec, err := keepertest.GetASpec(specname, "../../", nil, nil)
+	spec, err := specutils.GetASpec(specname, "../../", nil, nil)
 	require.NoError(t, err)
 
 	chainParser.SetPolicy(&plantypes.Policy{ChainPolicies: []plantypes.ChainPolicy{{ChainId: specname, Requirements: []plantypes.ChainRequirement{{Collection: spectypes.CollectionData{ApiInterface: "jsonrpc"}, Extensions: []string{"archive"}}}}}}, specname, "jsonrpc")

protocol/common/cobra_common.go

@@ -32,6 +32,7 @@ const (
 	// Disable relay retries when we get node errors.
 	// This feature is suppose to help with successful relays in some chains that return node errors on rare race conditions on the serviced chains.
 	DisableRetryOnNodeErrorsFlag = "disable-retry-on-node-error"
+	UseOfflineSpecFlag           = "use-offline-spec" // allows the user to manually load a spec providing a path, this is useful to test spec changes before they hit the blockchain
 )
 
 const (
@@ -55,6 +56,7 @@ type ConsumerCmdFlags struct {
 	DebugRelays                 bool          // enables debug mode for relays
 	DisableConflictTransactions bool          // disable conflict transactions
 	DisableRetryOnNodeErrors    bool          // disable retries on node errors
+	OfflineSpecPath             string        // path to the spec file, works only when bootstrapping a single chain.
 }
 
 // default rolling logs behavior (if enabled) will store 3 files each 100MB for up to 1 day every time.

protocol/rpcconsumer/rpcconsumer.go

@@ -29,6 +29,7 @@ import (
 	"github.com/lavanet/lava/v2/protocol/statetracker/updaters"
 	"github.com/lavanet/lava/v2/protocol/upgrade"
 	"github.com/lavanet/lava/v2/utils"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/utils/rand"
 	"github.com/lavanet/lava/v2/utils/sigs"
 	conflicttypes "github.com/lavanet/lava/v2/x/conflict/types"
@@ -214,8 +215,19 @@ func (rpcc *RPCConsumer) Start(ctx context.Context, options *rpcConsumerStartOpt
 			} else {
 				policyUpdaters.Store(rpcEndpoint.ChainID, updaters.NewPolicyUpdater(chainID, consumerStateTracker, consumerAddr.String(), chainParser, *rpcEndpoint))
 			}
-			// register for spec updates
-			err = rpcc.consumerStateTracker.RegisterForSpecUpdates(ctx, chainParser, *rpcEndpoint)
+
+			if options.cmdFlags.OfflineSpecPath != "" {
+				// offline spec mode.
+				parsedOfflineSpec, loadError := specutils.GetSpecFromPath(options.cmdFlags.OfflineSpecPath, rpcEndpoint.ChainID, nil, nil)
+				if loadError != nil {
+					err = utils.LavaFormatError("failed loading offline spec", err, utils.LogAttr("spec_path", options.cmdFlags.OfflineSpecPath), utils.LogAttr("spec_id", rpcEndpoint.ChainID))
+				}
+				utils.LavaFormatInfo("Loaded offline spec successfully", utils.LogAttr("spec_path", options.cmdFlags.OfflineSpecPath), utils.LogAttr("chain_id", parsedOfflineSpec.Index))
+				chainParser.SetSpec(parsedOfflineSpec)
+			} else {
+				// register for spec updates
+				err = rpcc.consumerStateTracker.RegisterForSpecUpdates(ctx, chainParser, *rpcEndpoint)
+			}
 			if err != nil {
 				err = utils.LavaFormatError("failed registering for spec updates", err, utils.Attribute{Key: "endpoint", Value: rpcEndpoint})
 				errCh <- err
@@ -561,7 +573,6 @@ rpcconsumer consumer_examples/full_consumer_example.yml --cache-be "127.0.0.1:77
 			}
 
 			maxConcurrentProviders := viper.GetUint(common.MaximumConcurrentProvidersFlagName)
-
 			consumerPropagatedFlags := common.ConsumerCmdFlags{
 				HeadersFlag:                 viper.GetString(common.CorsHeadersFlag),
 				CredentialsFlag:             viper.GetString(common.CorsCredentialsFlag),
@@ -573,6 +584,12 @@ rpcconsumer consumer_examples/full_consumer_example.yml --cache-be "127.0.0.1:77
 				DebugRelays:                 viper.GetBool(DebugRelaysFlagName),
 				DisableConflictTransactions: viper.GetBool(common.DisableConflictTransactionsFlag),
 				DisableRetryOnNodeErrors:    viper.GetBool(common.DisableRetryOnNodeErrorsFlag),
+				OfflineSpecPath:             viper.GetString(common.UseOfflineSpecFlag),
+			}
+
+			// validate user is does not provide multi chain setup when using the offline spec feature.
+			if consumerPropagatedFlags.OfflineSpecPath != "" && len(rpcEndpoints) > 1 {
+				utils.LavaFormatFatal("offline spec modifications are supported only in single chain bootstrapping", nil, utils.LogAttr("len(rpcEndpoints)", len(rpcEndpoints)), utils.LogAttr("rpcEndpoints", rpcEndpoints))
 			}
 
 			rpcConsumerSharedState := viper.GetBool(common.SharedStateFlag)
@@ -615,6 +632,7 @@ rpcconsumer consumer_examples/full_consumer_example.yml --cache-be "127.0.0.1:77
 	cmdRPCConsumer.Flags().Bool(common.DisableConflictTransactionsFlag, false, "disabling conflict transactions, this flag should not be used as it harms the network's data reliability and therefore the service.")
 	cmdRPCConsumer.Flags().DurationVar(&updaters.TimeOutForFetchingLavaBlocks, common.TimeOutForFetchingLavaBlocksFlag, time.Second*5, "setting the timeout for fetching lava blocks")
 	cmdRPCConsumer.Flags().Bool(common.DisableRetryOnNodeErrorsFlag, false, "Disable relay retries on node errors, prevent the rpcconsumer trying a different provider")
+	cmdRPCConsumer.Flags().String(common.UseOfflineSpecFlag, "", "load offline spec provided path to spec file, used to test specs before they are proposed on chain")
 
 	common.AddRollingLogConfig(cmdRPCConsumer)
 	return cmdRPCConsumer

testutil/common/tester.go

@@ -16,6 +16,7 @@ import (
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
 	testkeeper "github.com/lavanet/lava/v2/testutil/keeper"
 	"github.com/lavanet/lava/v2/utils"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/utils/lavaslices"
 	"github.com/lavanet/lava/v2/utils/sigs"
 	dualstakingante "github.com/lavanet/lava/v2/x/dualstaking/ante"
@@ -1127,7 +1128,7 @@ func (ts *Tester) SetupForTests(getToTopMostPath string, specId string, validato
 	}
 
 	sdkContext := sdk.UnwrapSDKContext(ts.Ctx)
-	spec, err := testkeeper.GetASpec(specId, getToTopMostPath, &sdkContext, &ts.Keepers.Spec)
+	spec, err := specutils.GetASpec(specId, getToTopMostPath, &sdkContext, &ts.Keepers.Spec)
 	if err != nil {
 		return err
 	}

testutil/keeper/spec.go → utils/keeper/spec.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"testing"
 
 	tmdb "github.com/cometbft/cometbft-db"
@@ -67,7 +66,20 @@ func specKeeper() (*keeper.Keeper, sdk.Context, error) {
 	return k, ctx, nil
 }
 
-func GetASpec(specIndex, getToTopMostPath string, ctxArg *sdk.Context, keeper *keeper.Keeper) (specRet spectypes.Spec, err error) {
+func decodeProposal(path string) (utils.SpecAddProposalJSON, error) {
+	proposal := utils.SpecAddProposalJSON{}
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return proposal, err
+	}
+	decoder := json.NewDecoder(bytes.NewReader(contents))
+	decoder.DisallowUnknownFields() // This will make the unmarshal fail if there are unused fields
+
+	err = decoder.Decode(&proposal)
+	return proposal, err
+}
+
+func GetSpecFromPath(path string, specIndex string, ctxArg *sdk.Context, keeper *keeper.Keeper) (specRet spectypes.Spec, err error) {
 	var ctx sdk.Context
 	if keeper == nil || ctxArg == nil {
 		keeper, ctx, err = specKeeper()
@@ -77,31 +89,48 @@ func GetASpec(specIndex, getToTopMostPath string, ctxArg *sdk.Context, keeper *k
 	} else {
 		ctx = *ctxArg
 	}
-	proposalFile := "./cookbook/specs/ibc.json,./cookbook/specs/cosmoswasm.json,./cookbook/specs/tendermint.json,./cookbook/specs/cosmossdk.json,./cookbook/specs/cosmossdk_full.json,./cookbook/specs/ethereum.json,./cookbook/specs/cosmoshub.json,./cookbook/specs/lava.json,./cookbook/specs/osmosis.json,./cookbook/specs/fantom.json,./cookbook/specs/celo.json,./cookbook/specs/optimism.json,./cookbook/specs/arbitrum.json,./cookbook/specs/starknet.json,./cookbook/specs/aptos.json,./cookbook/specs/juno.json,./cookbook/specs/polygon.json,./cookbook/specs/evmos.json,./cookbook/specs/base.json,./cookbook/specs/canto.json,./cookbook/specs/sui.json,./cookbook/specs/solana.json,./cookbook/specs/bsc.json,./cookbook/specs/axelar.json,./cookbook/specs/avalanche.json,./cookbook/specs/fvm.json"
-	for _, fileName := range strings.Split(proposalFile, ",") {
-		proposal := utils.SpecAddProposalJSON{}
 
-		contents, err := os.ReadFile(getToTopMostPath + fileName)
+	proposal, err := decodeProposal(path)
+	if err != nil {
+		return spectypes.Spec{}, err
+	}
+
+	for _, spec := range proposal.Proposal.Specs {
+		keeper.SetSpec(ctx, spec)
+		if specIndex != spec.Index {
+			continue
+		}
+		fullspec, err := keeper.ExpandSpec(ctx, spec)
 		if err != nil {
 			return spectypes.Spec{}, err
 		}
-		decoder := json.NewDecoder(bytes.NewReader(contents))
-		decoder.DisallowUnknownFields() // This will make the unmarshal fail if there are unused fields
+		return fullspec, nil
+	}
+	return spectypes.Spec{}, fmt.Errorf("spec not found %s", path)
+}
 
-		if err := decoder.Decode(&proposal); err != nil {
+func GetASpec(specIndex, getToTopMostPath string, ctxArg *sdk.Context, keeper *keeper.Keeper) (specRet spectypes.Spec, err error) {
+	var ctx sdk.Context
+	if keeper == nil || ctxArg == nil {
+		keeper, ctx, err = specKeeper()
+		if err != nil {
 			return spectypes.Spec{}, err
 		}
-
-		for _, spec := range proposal.Proposal.Specs {
-			keeper.SetSpec(ctx, spec)
-			if specIndex != spec.Index {
-				continue
-			}
-			fullspec, err := keeper.ExpandSpec(ctx, spec)
-			if err != nil {
-				return spectypes.Spec{}, err
-			}
-			return fullspec, nil
+	} else {
+		ctx = *ctxArg
+	}
+	proposalDirectory := "cookbook/specs/"
+	proposalFiles := []string{
+		"ibc.json", "cosmoswasm.json", "tendermint.json", "cosmossdk.json", "cosmossdk_full.json",
+		"ethereum.json", "cosmoshub.json", "lava.json", "osmosis.json", "fantom.json", "celo.json",
+		"optimism.json", "arbitrum.json", "starknet.json", "aptos.json", "juno.json", "polygon.json",
+		"evmos.json", "base.json", "canto.json", "sui.json", "solana.json", "bsc.json", "axelar.json",
+		"avalanche.json", "fvm.json", "near.json",
+	}
+	for _, fileName := range proposalFiles {
+		spec, err := GetSpecFromPath(getToTopMostPath+proposalDirectory+fileName, specIndex, &ctx, keeper)
+		if err == nil {
+			return spec, nil
 		}
 	}
 	return spectypes.Spec{}, fmt.Errorf("spec not found %s", specIndex)

x/pairing/keeper/pairing_test.go

@@ -10,6 +10,7 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/lavanet/lava/v2/testutil/common"
 	testkeeper "github.com/lavanet/lava/v2/testutil/keeper"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/utils/lavaslices"
 	"github.com/lavanet/lava/v2/utils/sigs"
 	epochstoragetypes "github.com/lavanet/lava/v2/x/epochstorage/types"
@@ -2212,7 +2213,7 @@ func TestMixBothExetensionAndAddonPairing(t *testing.T) {
 func TestMixSelectedProvidersAndArchivePairing(t *testing.T) {
 	ts := newTester(t)
 	ts.setupForPayments(1, 0, 0) // 1 provider, 0 client, default providers-to-pair
-	specEth, err := testkeeper.GetASpec("ETH1", "../../../", nil, nil)
+	specEth, err := specutils.GetASpec("ETH1", "../../../", nil, nil)
 	if err != nil {
 		require.NoError(t, err)
 	}

x/spec/ante/ante_test.go

@@ -13,7 +13,7 @@ import (
 	v1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1"
 	"github.com/cosmos/gogoproto/proto"
 	"github.com/lavanet/lava/v2/app"
-	testkeeper "github.com/lavanet/lava/v2/testutil/keeper"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	plantypes "github.com/lavanet/lava/v2/x/plans/types"
 	"github.com/lavanet/lava/v2/x/spec/ante"
 	spectypes "github.com/lavanet/lava/v2/x/spec/types"
@@ -181,7 +181,7 @@ func TestNewExpeditedProposalFilterAnteDecorator(t *testing.T) {
 		tt := tt
 
 		t.Run(tt.name, func(t *testing.T) {
-			k, ctx := testkeeper.SpecKeeper(t)
+			k, ctx := specutils.SpecKeeper(t)
 			params := spectypes.DefaultParams()
 			params.AllowlistedExpeditedMsgs = []string{
 				proto.MessageName(&banktypes.MsgSend{}),

x/spec/genesis_test.go

@@ -6,8 +6,8 @@ import (
 	types2 "github.com/cosmos/cosmos-sdk/x/auth/types"
 	"github.com/cosmos/gogoproto/proto"
 
-	keepertest "github.com/lavanet/lava/v2/testutil/keeper"
 	"github.com/lavanet/lava/v2/testutil/nullify"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/x/spec"
 	"github.com/lavanet/lava/v2/x/spec/types"
 	"github.com/stretchr/testify/require"
@@ -32,7 +32,7 @@ func TestGenesis(t *testing.T) {
 		// this line is used by starport scaffolding # genesis/test/state
 	}
 
-	k, ctx := keepertest.SpecKeeper(t)
+	k, ctx := specutils.SpecKeeper(t)
 	spec.InitGenesis(ctx, *k, genesisState)
 	got := spec.ExportGenesis(ctx, *k)
 	require.NotNil(t, got)

x/spec/keeper/grpc_query_params_test.go

@@ -4,13 +4,13 @@ import (
 	"testing"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
-	testkeeper "github.com/lavanet/lava/v2/testutil/keeper"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/x/spec/types"
 	"github.com/stretchr/testify/require"
 )
 
 func TestParamsQuery(t *testing.T) {
-	keeper, ctx := testkeeper.SpecKeeper(t)
+	keeper, ctx := specutils.SpecKeeper(t)
 	wctx := sdk.WrapSDKContext(ctx)
 	params := types.DefaultParams()
 	keeper.SetParams(ctx, params)

x/spec/keeper/grpc_query_spec_test.go

@@ -10,16 +10,16 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	keepertest "github.com/lavanet/lava/v2/testutil/keeper"
 	"github.com/lavanet/lava/v2/testutil/nullify"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/x/spec/types"
 )
 
 // Prevent strconv unused error
 var _ = strconv.IntSize
 
 func TestSpecQuerySingle(t *testing.T) {
-	keeper, ctx := keepertest.SpecKeeper(t)
+	keeper, ctx := specutils.SpecKeeper(t)
 	wctx := sdk.WrapSDKContext(ctx)
 	msgs := createNSpec(keeper, ctx, 2)
 	for _, tc := range []struct {
@@ -70,7 +70,7 @@ func TestSpecQuerySingle(t *testing.T) {
 }
 
 func TestSpecQuerySingleRaw(t *testing.T) {
-	keeper, ctx := keepertest.SpecKeeper(t)
+	keeper, ctx := specutils.SpecKeeper(t)
 	wctx := sdk.WrapSDKContext(ctx)
 	msgs := createNSpec(keeper, ctx, 2)
 
@@ -98,7 +98,7 @@ func TestSpecQuerySingleRaw(t *testing.T) {
 }
 
 func TestSpecQueryPaginated(t *testing.T) {
-	keeper, ctx := keepertest.SpecKeeper(t)
+	keeper, ctx := specutils.SpecKeeper(t)
 	wctx := sdk.WrapSDKContext(ctx)
 	msgs := createNSpec(keeper, ctx, 5)
 

x/spec/keeper/msg_server_test.go

@@ -5,12 +5,12 @@ import (
 	"testing"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
-	keepertest "github.com/lavanet/lava/v2/testutil/keeper"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/x/spec/keeper"
 	"github.com/lavanet/lava/v2/x/spec/types"
 )
 
 func setupMsgServer(t testing.TB) (types.MsgServer, context.Context) {
-	k, ctx := keepertest.SpecKeeper(t)
+	k, ctx := specutils.SpecKeeper(t)
 	return keeper.NewMsgServerImpl(*k), sdk.WrapSDKContext(ctx)
 }

x/spec/keeper/params_test.go

@@ -3,13 +3,13 @@ package keeper_test
 import (
 	"testing"
 
-	testkeeper "github.com/lavanet/lava/v2/testutil/keeper"
+	specutils "github.com/lavanet/lava/v2/utils/keeper"
 	"github.com/lavanet/lava/v2/x/spec/types"
 	"github.com/stretchr/testify/require"
 )
 
 func TestGetParams(t *testing.T) {
-	k, ctx := testkeeper.SpecKeeper(t)
+	k, ctx := specutils.SpecKeeper(t)
 	params := types.DefaultParams()
 
 	k.SetParams(ctx, params)