diff --git a/protocol/chainlib/common_test_utils.go b/protocol/chainlib/common_test_utils.go index 2782799b23..1e4db5dc47 100644 --- a/protocol/chainlib/common_test_utils.go +++ b/protocol/chainlib/common_test_utils.go @@ -26,6 +26,7 @@ import ( "github.com/lavanet/lava/v2/protocol/lavasession" testcommon "github.com/lavanet/lava/v2/testutil/common" keepertest "github.com/lavanet/lava/v2/testutil/keeper" + specutils "github.com/lavanet/lava/v2/utils/keeper" plantypes "github.com/lavanet/lava/v2/x/plans/types" spectypes "github.com/lavanet/lava/v2/x/spec/types" "github.com/stretchr/testify/require" @@ -126,7 +127,7 @@ func CreateChainLibMocks( ) (cpar ChainParser, crout ChainRouter, cfetc chaintracker.ChainFetcher, closeServer func(), endpointRet *lavasession.RPCProviderEndpoint, errRet error) { utils.SetGlobalLoggingLevel("debug") closeServer = nil - spec, err := keepertest.GetASpec(specIndex, getToTopMostPath, nil, nil) + spec, err := specutils.GetASpec(specIndex, getToTopMostPath, nil, nil) if err != nil { return nil, nil, nil, nil, nil, err } @@ -250,7 +251,7 @@ func SetupForTests(t *testing.T, numOfProviders int, specID string, getToTopMost ts.Providers = append(ts.Providers, testcommon.CreateNewAccount(ts.Ctx, *ts.Keepers, balance)) } sdkContext := sdk.UnwrapSDKContext(ts.Ctx) - spec, err := keepertest.GetASpec(specID, getToTopMostPath, &sdkContext, &ts.Keepers.Spec) + spec, err := specutils.GetASpec(specID, getToTopMostPath, &sdkContext, &ts.Keepers.Spec) if err != nil { require.NoError(t, err) } diff --git a/protocol/chainlib/jsonRPC_test.go b/protocol/chainlib/jsonRPC_test.go index a110b22bca..15f9db0dc2 100644 --- a/protocol/chainlib/jsonRPC_test.go +++ b/protocol/chainlib/jsonRPC_test.go @@ -13,7 +13,7 @@ import ( "github.com/lavanet/lava/v2/protocol/chainlib/chainproxy/rpcInterfaceMessages" "github.com/lavanet/lava/v2/protocol/chainlib/extensionslib" "github.com/lavanet/lava/v2/protocol/common" - keepertest "github.com/lavanet/lava/v2/testutil/keeper" + specutils "github.com/lavanet/lava/v2/utils/keeper" plantypes "github.com/lavanet/lava/v2/x/plans/types" spectypes "github.com/lavanet/lava/v2/x/spec/types" "github.com/stretchr/testify/assert" @@ -253,7 +253,7 @@ func TestExtensions(t *testing.T) { configuredExtensions := map[string]struct{}{ "archive": {}, } - spec, err := keepertest.GetASpec(specname, "../../", nil, nil) + spec, err := specutils.GetASpec(specname, "../../", nil, nil) require.NoError(t, err) chainParser.SetPolicy(&plantypes.Policy{ChainPolicies: []plantypes.ChainPolicy{{ChainId: specname, Requirements: []plantypes.ChainRequirement{{Collection: spectypes.CollectionData{ApiInterface: "jsonrpc"}, Extensions: []string{"archive"}}}}}}, specname, "jsonrpc") diff --git a/protocol/common/cobra_common.go b/protocol/common/cobra_common.go index 19a64487d2..d7daa95e59 100644 --- a/protocol/common/cobra_common.go +++ b/protocol/common/cobra_common.go @@ -32,6 +32,7 @@ const ( // Disable relay retries when we get node errors. // This feature is suppose to help with successful relays in some chains that return node errors on rare race conditions on the serviced chains. DisableRetryOnNodeErrorsFlag = "disable-retry-on-node-error" + UseOfflineSpecFlag = "use-offline-spec" // allows the user to manually load a spec providing a path, this is useful to test spec changes before they hit the blockchain ) const ( @@ -55,6 +56,7 @@ type ConsumerCmdFlags struct { DebugRelays bool // enables debug mode for relays DisableConflictTransactions bool // disable conflict transactions DisableRetryOnNodeErrors bool // disable retries on node errors + OfflineSpecPath string // path to the spec file, works only when bootstrapping a single chain. } // default rolling logs behavior (if enabled) will store 3 files each 100MB for up to 1 day every time. diff --git a/protocol/rpcconsumer/rpcconsumer.go b/protocol/rpcconsumer/rpcconsumer.go index 8cee4014e9..c5b5a32e5b 100644 --- a/protocol/rpcconsumer/rpcconsumer.go +++ b/protocol/rpcconsumer/rpcconsumer.go @@ -29,6 +29,7 @@ import ( "github.com/lavanet/lava/v2/protocol/statetracker/updaters" "github.com/lavanet/lava/v2/protocol/upgrade" "github.com/lavanet/lava/v2/utils" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/utils/rand" "github.com/lavanet/lava/v2/utils/sigs" conflicttypes "github.com/lavanet/lava/v2/x/conflict/types" @@ -214,8 +215,19 @@ func (rpcc *RPCConsumer) Start(ctx context.Context, options *rpcConsumerStartOpt } else { policyUpdaters.Store(rpcEndpoint.ChainID, updaters.NewPolicyUpdater(chainID, consumerStateTracker, consumerAddr.String(), chainParser, *rpcEndpoint)) } - // register for spec updates - err = rpcc.consumerStateTracker.RegisterForSpecUpdates(ctx, chainParser, *rpcEndpoint) + + if options.cmdFlags.OfflineSpecPath != "" { + // offline spec mode. + parsedOfflineSpec, loadError := specutils.GetSpecFromPath(options.cmdFlags.OfflineSpecPath, rpcEndpoint.ChainID, nil, nil) + if loadError != nil { + err = utils.LavaFormatError("failed loading offline spec", err, utils.LogAttr("spec_path", options.cmdFlags.OfflineSpecPath), utils.LogAttr("spec_id", rpcEndpoint.ChainID)) + } + utils.LavaFormatInfo("Loaded offline spec successfully", utils.LogAttr("spec_path", options.cmdFlags.OfflineSpecPath), utils.LogAttr("chain_id", parsedOfflineSpec.Index)) + chainParser.SetSpec(parsedOfflineSpec) + } else { + // register for spec updates + err = rpcc.consumerStateTracker.RegisterForSpecUpdates(ctx, chainParser, *rpcEndpoint) + } if err != nil { err = utils.LavaFormatError("failed registering for spec updates", err, utils.Attribute{Key: "endpoint", Value: rpcEndpoint}) errCh <- err @@ -561,7 +573,6 @@ rpcconsumer consumer_examples/full_consumer_example.yml --cache-be "127.0.0.1:77 } maxConcurrentProviders := viper.GetUint(common.MaximumConcurrentProvidersFlagName) - consumerPropagatedFlags := common.ConsumerCmdFlags{ HeadersFlag: viper.GetString(common.CorsHeadersFlag), CredentialsFlag: viper.GetString(common.CorsCredentialsFlag), @@ -573,6 +584,12 @@ rpcconsumer consumer_examples/full_consumer_example.yml --cache-be "127.0.0.1:77 DebugRelays: viper.GetBool(DebugRelaysFlagName), DisableConflictTransactions: viper.GetBool(common.DisableConflictTransactionsFlag), DisableRetryOnNodeErrors: viper.GetBool(common.DisableRetryOnNodeErrorsFlag), + OfflineSpecPath: viper.GetString(common.UseOfflineSpecFlag), + } + + // validate user is does not provide multi chain setup when using the offline spec feature. + if consumerPropagatedFlags.OfflineSpecPath != "" && len(rpcEndpoints) > 1 { + utils.LavaFormatFatal("offline spec modifications are supported only in single chain bootstrapping", nil, utils.LogAttr("len(rpcEndpoints)", len(rpcEndpoints)), utils.LogAttr("rpcEndpoints", rpcEndpoints)) } rpcConsumerSharedState := viper.GetBool(common.SharedStateFlag) @@ -615,6 +632,7 @@ rpcconsumer consumer_examples/full_consumer_example.yml --cache-be "127.0.0.1:77 cmdRPCConsumer.Flags().Bool(common.DisableConflictTransactionsFlag, false, "disabling conflict transactions, this flag should not be used as it harms the network's data reliability and therefore the service.") cmdRPCConsumer.Flags().DurationVar(&updaters.TimeOutForFetchingLavaBlocks, common.TimeOutForFetchingLavaBlocksFlag, time.Second*5, "setting the timeout for fetching lava blocks") cmdRPCConsumer.Flags().Bool(common.DisableRetryOnNodeErrorsFlag, false, "Disable relay retries on node errors, prevent the rpcconsumer trying a different provider") + cmdRPCConsumer.Flags().String(common.UseOfflineSpecFlag, "", "load offline spec provided path to spec file, used to test specs before they are proposed on chain") common.AddRollingLogConfig(cmdRPCConsumer) return cmdRPCConsumer diff --git a/testutil/common/tester.go b/testutil/common/tester.go index fd5ca31827..d6437ac909 100644 --- a/testutil/common/tester.go +++ b/testutil/common/tester.go @@ -16,6 +16,7 @@ import ( stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" testkeeper "github.com/lavanet/lava/v2/testutil/keeper" "github.com/lavanet/lava/v2/utils" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/utils/lavaslices" "github.com/lavanet/lava/v2/utils/sigs" dualstakingante "github.com/lavanet/lava/v2/x/dualstaking/ante" @@ -1127,7 +1128,7 @@ func (ts *Tester) SetupForTests(getToTopMostPath string, specId string, validato } sdkContext := sdk.UnwrapSDKContext(ts.Ctx) - spec, err := testkeeper.GetASpec(specId, getToTopMostPath, &sdkContext, &ts.Keepers.Spec) + spec, err := specutils.GetASpec(specId, getToTopMostPath, &sdkContext, &ts.Keepers.Spec) if err != nil { return err } diff --git a/testutil/keeper/spec.go b/utils/keeper/spec.go similarity index 56% rename from testutil/keeper/spec.go rename to utils/keeper/spec.go index 3574f15040..f76965c5a4 100644 --- a/testutil/keeper/spec.go +++ b/utils/keeper/spec.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "os" - "strings" "testing" tmdb "github.com/cometbft/cometbft-db" @@ -67,7 +66,20 @@ func specKeeper() (*keeper.Keeper, sdk.Context, error) { return k, ctx, nil } -func GetASpec(specIndex, getToTopMostPath string, ctxArg *sdk.Context, keeper *keeper.Keeper) (specRet spectypes.Spec, err error) { +func decodeProposal(path string) (utils.SpecAddProposalJSON, error) { + proposal := utils.SpecAddProposalJSON{} + contents, err := os.ReadFile(path) + if err != nil { + return proposal, err + } + decoder := json.NewDecoder(bytes.NewReader(contents)) + decoder.DisallowUnknownFields() // This will make the unmarshal fail if there are unused fields + + err = decoder.Decode(&proposal) + return proposal, err +} + +func GetSpecFromPath(path string, specIndex string, ctxArg *sdk.Context, keeper *keeper.Keeper) (specRet spectypes.Spec, err error) { var ctx sdk.Context if keeper == nil || ctxArg == nil { keeper, ctx, err = specKeeper() @@ -77,31 +89,48 @@ func GetASpec(specIndex, getToTopMostPath string, ctxArg *sdk.Context, keeper *k } else { ctx = *ctxArg } - proposalFile := "./cookbook/specs/ibc.json,./cookbook/specs/cosmoswasm.json,./cookbook/specs/tendermint.json,./cookbook/specs/cosmossdk.json,./cookbook/specs/cosmossdk_full.json,./cookbook/specs/ethereum.json,./cookbook/specs/cosmoshub.json,./cookbook/specs/lava.json,./cookbook/specs/osmosis.json,./cookbook/specs/fantom.json,./cookbook/specs/celo.json,./cookbook/specs/optimism.json,./cookbook/specs/arbitrum.json,./cookbook/specs/starknet.json,./cookbook/specs/aptos.json,./cookbook/specs/juno.json,./cookbook/specs/polygon.json,./cookbook/specs/evmos.json,./cookbook/specs/base.json,./cookbook/specs/canto.json,./cookbook/specs/sui.json,./cookbook/specs/solana.json,./cookbook/specs/bsc.json,./cookbook/specs/axelar.json,./cookbook/specs/avalanche.json,./cookbook/specs/fvm.json" - for _, fileName := range strings.Split(proposalFile, ",") { - proposal := utils.SpecAddProposalJSON{} - contents, err := os.ReadFile(getToTopMostPath + fileName) + proposal, err := decodeProposal(path) + if err != nil { + return spectypes.Spec{}, err + } + + for _, spec := range proposal.Proposal.Specs { + keeper.SetSpec(ctx, spec) + if specIndex != spec.Index { + continue + } + fullspec, err := keeper.ExpandSpec(ctx, spec) if err != nil { return spectypes.Spec{}, err } - decoder := json.NewDecoder(bytes.NewReader(contents)) - decoder.DisallowUnknownFields() // This will make the unmarshal fail if there are unused fields + return fullspec, nil + } + return spectypes.Spec{}, fmt.Errorf("spec not found %s", path) +} - if err := decoder.Decode(&proposal); err != nil { +func GetASpec(specIndex, getToTopMostPath string, ctxArg *sdk.Context, keeper *keeper.Keeper) (specRet spectypes.Spec, err error) { + var ctx sdk.Context + if keeper == nil || ctxArg == nil { + keeper, ctx, err = specKeeper() + if err != nil { return spectypes.Spec{}, err } - - for _, spec := range proposal.Proposal.Specs { - keeper.SetSpec(ctx, spec) - if specIndex != spec.Index { - continue - } - fullspec, err := keeper.ExpandSpec(ctx, spec) - if err != nil { - return spectypes.Spec{}, err - } - return fullspec, nil + } else { + ctx = *ctxArg + } + proposalDirectory := "cookbook/specs/" + proposalFiles := []string{ + "ibc.json", "cosmoswasm.json", "tendermint.json", "cosmossdk.json", "cosmossdk_full.json", + "ethereum.json", "cosmoshub.json", "lava.json", "osmosis.json", "fantom.json", "celo.json", + "optimism.json", "arbitrum.json", "starknet.json", "aptos.json", "juno.json", "polygon.json", + "evmos.json", "base.json", "canto.json", "sui.json", "solana.json", "bsc.json", "axelar.json", + "avalanche.json", "fvm.json", "near.json", + } + for _, fileName := range proposalFiles { + spec, err := GetSpecFromPath(getToTopMostPath+proposalDirectory+fileName, specIndex, &ctx, keeper) + if err == nil { + return spec, nil } } return spectypes.Spec{}, fmt.Errorf("spec not found %s", specIndex) diff --git a/x/pairing/keeper/pairing_test.go b/x/pairing/keeper/pairing_test.go index 74165525af..a9cf57336b 100644 --- a/x/pairing/keeper/pairing_test.go +++ b/x/pairing/keeper/pairing_test.go @@ -10,6 +10,7 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" "github.com/lavanet/lava/v2/testutil/common" testkeeper "github.com/lavanet/lava/v2/testutil/keeper" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/utils/lavaslices" "github.com/lavanet/lava/v2/utils/sigs" epochstoragetypes "github.com/lavanet/lava/v2/x/epochstorage/types" @@ -2212,7 +2213,7 @@ func TestMixBothExetensionAndAddonPairing(t *testing.T) { func TestMixSelectedProvidersAndArchivePairing(t *testing.T) { ts := newTester(t) ts.setupForPayments(1, 0, 0) // 1 provider, 0 client, default providers-to-pair - specEth, err := testkeeper.GetASpec("ETH1", "../../../", nil, nil) + specEth, err := specutils.GetASpec("ETH1", "../../../", nil, nil) if err != nil { require.NoError(t, err) } diff --git a/x/spec/ante/ante_test.go b/x/spec/ante/ante_test.go index 9047af0f1d..82571e5413 100644 --- a/x/spec/ante/ante_test.go +++ b/x/spec/ante/ante_test.go @@ -13,7 +13,7 @@ import ( v1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1" "github.com/cosmos/gogoproto/proto" "github.com/lavanet/lava/v2/app" - testkeeper "github.com/lavanet/lava/v2/testutil/keeper" + specutils "github.com/lavanet/lava/v2/utils/keeper" plantypes "github.com/lavanet/lava/v2/x/plans/types" "github.com/lavanet/lava/v2/x/spec/ante" spectypes "github.com/lavanet/lava/v2/x/spec/types" @@ -181,7 +181,7 @@ func TestNewExpeditedProposalFilterAnteDecorator(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { - k, ctx := testkeeper.SpecKeeper(t) + k, ctx := specutils.SpecKeeper(t) params := spectypes.DefaultParams() params.AllowlistedExpeditedMsgs = []string{ proto.MessageName(&banktypes.MsgSend{}), diff --git a/x/spec/genesis_test.go b/x/spec/genesis_test.go index 8604f72bc1..9faaa7e9ae 100644 --- a/x/spec/genesis_test.go +++ b/x/spec/genesis_test.go @@ -6,8 +6,8 @@ import ( types2 "github.com/cosmos/cosmos-sdk/x/auth/types" "github.com/cosmos/gogoproto/proto" - keepertest "github.com/lavanet/lava/v2/testutil/keeper" "github.com/lavanet/lava/v2/testutil/nullify" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/x/spec" "github.com/lavanet/lava/v2/x/spec/types" "github.com/stretchr/testify/require" @@ -32,7 +32,7 @@ func TestGenesis(t *testing.T) { // this line is used by starport scaffolding # genesis/test/state } - k, ctx := keepertest.SpecKeeper(t) + k, ctx := specutils.SpecKeeper(t) spec.InitGenesis(ctx, *k, genesisState) got := spec.ExportGenesis(ctx, *k) require.NotNil(t, got) diff --git a/x/spec/keeper/grpc_query_params_test.go b/x/spec/keeper/grpc_query_params_test.go index ada5f2d3f0..5d94e82188 100644 --- a/x/spec/keeper/grpc_query_params_test.go +++ b/x/spec/keeper/grpc_query_params_test.go @@ -4,13 +4,13 @@ import ( "testing" sdk "github.com/cosmos/cosmos-sdk/types" - testkeeper "github.com/lavanet/lava/v2/testutil/keeper" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/x/spec/types" "github.com/stretchr/testify/require" ) func TestParamsQuery(t *testing.T) { - keeper, ctx := testkeeper.SpecKeeper(t) + keeper, ctx := specutils.SpecKeeper(t) wctx := sdk.WrapSDKContext(ctx) params := types.DefaultParams() keeper.SetParams(ctx, params) diff --git a/x/spec/keeper/grpc_query_spec_test.go b/x/spec/keeper/grpc_query_spec_test.go index 0b1e33d1a8..ad97b52fbf 100644 --- a/x/spec/keeper/grpc_query_spec_test.go +++ b/x/spec/keeper/grpc_query_spec_test.go @@ -10,8 +10,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - keepertest "github.com/lavanet/lava/v2/testutil/keeper" "github.com/lavanet/lava/v2/testutil/nullify" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/x/spec/types" ) @@ -19,7 +19,7 @@ import ( var _ = strconv.IntSize func TestSpecQuerySingle(t *testing.T) { - keeper, ctx := keepertest.SpecKeeper(t) + keeper, ctx := specutils.SpecKeeper(t) wctx := sdk.WrapSDKContext(ctx) msgs := createNSpec(keeper, ctx, 2) for _, tc := range []struct { @@ -70,7 +70,7 @@ func TestSpecQuerySingle(t *testing.T) { } func TestSpecQuerySingleRaw(t *testing.T) { - keeper, ctx := keepertest.SpecKeeper(t) + keeper, ctx := specutils.SpecKeeper(t) wctx := sdk.WrapSDKContext(ctx) msgs := createNSpec(keeper, ctx, 2) @@ -98,7 +98,7 @@ func TestSpecQuerySingleRaw(t *testing.T) { } func TestSpecQueryPaginated(t *testing.T) { - keeper, ctx := keepertest.SpecKeeper(t) + keeper, ctx := specutils.SpecKeeper(t) wctx := sdk.WrapSDKContext(ctx) msgs := createNSpec(keeper, ctx, 5) diff --git a/x/spec/keeper/msg_server_test.go b/x/spec/keeper/msg_server_test.go index 241f721f95..b15805f52b 100644 --- a/x/spec/keeper/msg_server_test.go +++ b/x/spec/keeper/msg_server_test.go @@ -5,12 +5,12 @@ import ( "testing" sdk "github.com/cosmos/cosmos-sdk/types" - keepertest "github.com/lavanet/lava/v2/testutil/keeper" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/x/spec/keeper" "github.com/lavanet/lava/v2/x/spec/types" ) func setupMsgServer(t testing.TB) (types.MsgServer, context.Context) { - k, ctx := keepertest.SpecKeeper(t) + k, ctx := specutils.SpecKeeper(t) return keeper.NewMsgServerImpl(*k), sdk.WrapSDKContext(ctx) } diff --git a/x/spec/keeper/params_test.go b/x/spec/keeper/params_test.go index 089696024e..429f0d7410 100644 --- a/x/spec/keeper/params_test.go +++ b/x/spec/keeper/params_test.go @@ -3,13 +3,13 @@ package keeper_test import ( "testing" - testkeeper "github.com/lavanet/lava/v2/testutil/keeper" + specutils "github.com/lavanet/lava/v2/utils/keeper" "github.com/lavanet/lava/v2/x/spec/types" "github.com/stretchr/testify/require" ) func TestGetParams(t *testing.T) { - k, ctx := testkeeper.SpecKeeper(t) + k, ctx := specutils.SpecKeeper(t) params := types.DefaultParams() k.SetParams(ctx, params)