diff --git a/core/capabilities/integration_tests/framework/capabilities_registry.go b/core/capabilities/integration_tests/framework/capabilities_registry.go index 604c1d082d8..7acabfdcc23 100644 --- a/core/capabilities/integration_tests/framework/capabilities_registry.go +++ b/core/capabilities/integration_tests/framework/capabilities_registry.go @@ -65,6 +65,8 @@ func (r *CapabilitiesRegistry) getAddress() common.Address { type capability struct { donCapabilityConfig *pb.CapabilityConfig registryConfig kcr.CapabilitiesRegistryCapability + // internalOnly is true if the capability is published in the registry but not made available outside the DON in which it runs + internalOnly bool } // SetupDON sets up a new DON with the given capabilities and returns the DON ID diff --git a/core/capabilities/integration_tests/framework/don.go b/core/capabilities/integration_tests/framework/don.go index 1cb38c1bf71..62f89fe0afc 100644 --- a/core/capabilities/integration_tests/framework/don.go +++ b/core/capabilities/integration_tests/framework/don.go @@ -2,6 +2,7 @@ package framework import ( "context" + "encoding/hex" "fmt" "strconv" "testing" @@ -11,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/durationpb" + "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" "github.com/smartcontractkit/chainlink/v2/core/services/registrysyncer" @@ -40,13 +42,13 @@ import ( type DonContext struct { EthBlockchain *EthBlockchain - p2pNetwork *MockRageP2PNetwork + p2pNetwork *FakeRageP2PNetwork capabilityRegistry *CapabilitiesRegistry } func CreateDonContext(ctx context.Context, t *testing.T) DonContext { ethBlockchain := NewEthBlockchain(t, 1000, 1*time.Second) - rageP2PNetwork := NewMockRageP2PNetwork(t, 1000) + rageP2PNetwork := NewFakeRageP2PNetwork(ctx, t, 1000) capabilitiesRegistry := NewCapabilitiesRegistry(ctx, t, ethBlockchain) servicetest.Run(t, rageP2PNetwork) @@ -54,6 +56,31 @@ func CreateDonContext(ctx context.Context, t *testing.T) DonContext { return DonContext{EthBlockchain: ethBlockchain, p2pNetwork: rageP2PNetwork, capabilityRegistry: capabilitiesRegistry} } +func (c DonContext) WaitForCapabilitiesToBeExposed(t *testing.T, dons ...*DON) { + + allExpectedCapabilities := make(map[CapabilityRegistration]bool) + for _, don := range dons { + caps, err := don.GetExternalCapabilities() + require.NoError(t, err) + for k, v := range caps { + allExpectedCapabilities[k] = v + } + } + + require.Eventually(t, func() bool { + registrations := c.p2pNetwork.GetCapabilityRegistrations() + + for k := range allExpectedCapabilities { + if _, ok := registrations[k]; !ok { + return false + } + } + + return true + }, 1*time.Minute, 1*time.Second, "timeout waiting for capabilities to be exposed") + +} + type capabilityNode struct { *cltest.TestApplication registry *capabilities.Registry @@ -64,12 +91,13 @@ type capabilityNode struct { } type DON struct { + services.StateMachine t *testing.T config DonConfiguration lggr logger.Logger nodes []*capabilityNode standardCapabilityJobs []*job.Job - externalCapabilities []capability + publishedCapabilities []capability capabilitiesRegistry *CapabilitiesRegistry nodeConfigModifiers []func(c *chainlink.Config, node *capabilityNode) @@ -84,10 +112,13 @@ func NewDON(ctx context.Context, t *testing.T, lggr logger.Logger, donConfig Don dependentDONs []commoncap.DON, donContext DonContext, supportsOCR bool) *DON { don := &DON{t: t, lggr: lggr.Named(donConfig.name), config: donConfig, capabilitiesRegistry: donContext.capabilityRegistry} + protocolRoundInterval := 1 * time.Second + var newOracleFactoryFn standardcapabilities.NewOracleFactoryFn - var libOcr *MockLibOCR + var libOcr *FakeLibOCR if supportsOCR { - libOcr = NewMockLibOCR(t, lggr, donConfig.F, 1*time.Second) + // This is required to support the non standard OCR3 capability - will be removed when required OCR3 behaviour is implemented as standard capabilities + libOcr = NewFakeLibOCR(t, lggr, donConfig.F, protocolRoundInterval) servicetest.Run(t, libOcr) } @@ -110,7 +141,8 @@ func NewDON(ctx context.Context, t *testing.T, lggr logger.Logger, donConfig Don don.nodes = append(don.nodes, cn) if supportsOCR { - factory := newMockLibOcrOracleFactory(libOcr, donConfig.KeyBundles[i], len(donConfig.Members), int(donConfig.F)) + factory := newFakeOracleFactoryFactory(ctx, t, lggr, donConfig.KeyBundles[i], len(donConfig.Members), int(donConfig.F), + protocolRoundInterval) newOracleFactoryFn = factory.NewOracleFactory } @@ -134,12 +166,10 @@ func NewDON(ctx context.Context, t *testing.T, lggr logger.Logger, donConfig Don // Initialise must be called after all capabilities have been added to the DONs and before Start is called func (d *DON) Initialise() { - if len(d.externalCapabilities) > 0 { - id := d.capabilitiesRegistry.setupDON(d.config, d.externalCapabilities) + id := d.capabilitiesRegistry.setupDON(d.config, d.publishedCapabilities) - //nolint:gosec // disable G115 - d.config.DON.ID = uint32(id) - } + //nolint:gosec // disable G115 + d.config.DON.ID = uint32(id) } func (d *DON) GetID() uint32 { @@ -150,6 +180,31 @@ func (d *DON) GetID() uint32 { return d.config.ID } +func (d *DON) GetExternalCapabilities() (map[CapabilityRegistration]bool, error) { + + result := map[CapabilityRegistration]bool{} + for _, publishedCapability := range d.publishedCapabilities { + if publishedCapability.internalOnly { + continue + } + + for _, node := range d.nodes { + + peerIDBytes, err := peerIDToBytes(node.peerID.PeerID) + if err != nil { + return nil, fmt.Errorf("failed to convert peer ID to bytes: %w", err) + } + result[CapabilityRegistration{ + nodePeerID: hex.EncodeToString(peerIDBytes[:]), + capabilityID: publishedCapability.registryConfig.LabelledName + "@" + publishedCapability.registryConfig.Version, + capabilityDonID: d.GetID(), + }] = true + } + } + + return result, nil +} + func (d *DON) GetConfigVersion() uint32 { return d.config.ConfigVersion } @@ -162,20 +217,24 @@ func (d *DON) GetPeerIDs() []peer { return d.config.peerIDs } -func (d *DON) Start(ctx context.Context, t *testing.T) { +func (d *DON) Start(ctx context.Context) error { for _, triggerFactory := range d.triggerFactories { for _, node := range d.nodes { - trigger := triggerFactory.CreateNewTrigger(t) - err := node.registry.Add(ctx, trigger) - require.NoError(t, err) + trigger := triggerFactory.CreateNewTrigger(d.t) + if err := node.registry.Add(ctx, trigger); err != nil { + return fmt.Errorf("failed to add trigger: %w", err) + } + } } for _, targetFactory := range d.targetFactories { for _, node := range d.nodes { - target := targetFactory.CreateNewTarget(t) - err := node.registry.Add(ctx, target) - require.NoError(t, err) + target := targetFactory.CreateNewTarget(d.t) + if err := node.registry.Add(ctx, target); err != nil { + return fmt.Errorf("failed to add target: %w", err) + } + } } @@ -184,18 +243,32 @@ func (d *DON) Start(ctx context.Context, t *testing.T) { } if d.addOCR3NonStandardCapability { - libocr := NewMockLibOCR(t, d.lggr, d.config.F, 1*time.Second) - servicetest.Run(t, libocr) + libocr := NewFakeLibOCR(d.t, d.lggr, d.config.F, 1*time.Second) + servicetest.Run(d.t, libocr) for _, node := range d.nodes { - addOCR3Capability(ctx, t, d.lggr, node.registry, libocr, d.config.F, node.KeyBundle) + addOCR3Capability(ctx, d.t, d.lggr, node.registry, libocr, d.config.F, node.KeyBundle) } } for _, capabilityJob := range d.standardCapabilityJobs { - err := d.AddJob(ctx, capabilityJob) - require.NoError(t, err) + if err := d.AddJob(ctx, capabilityJob); err != nil { + return fmt.Errorf("failed to add standard capability job: %w", err) + } + + } + + return nil +} + +func (d *DON) Close() error { + for _, node := range d.nodes { + if err := node.Stop(); err != nil { + return fmt.Errorf("failed to stop node: %w", err) + } } + + return nil } const StandardCapabilityTemplateJobSpec = ` @@ -203,7 +276,7 @@ type = "standardcapabilities" schemaVersion = 1 name = "%s" command="%s" -config="%s" +config=%s ` func (d *DON) AddStandardCapability(name string, command string, config string) { @@ -214,11 +287,30 @@ func (d *DON) AddStandardCapability(name string, command string, config string) d.standardCapabilityJobs = append(d.standardCapabilityJobs, &capabilitiesSpecJob) } +func (d *DON) AddPublishedStandardCapability(name string, command string, config string, + defaultCapabilityRequestConfig *pb.CapabilityConfig, + registryConfig kcr.CapabilitiesRegistryCapability) { + spec := fmt.Sprintf(StandardCapabilityTemplateJobSpec, name, command, config) + capabilitiesSpecJob, err := standardcapabilities.ValidatedStandardCapabilitiesSpec(spec) + require.NoError(d.t, err) + + d.standardCapabilityJobs = append(d.standardCapabilityJobs, &capabilitiesSpecJob) + + d.publishedCapabilities = append(d.publishedCapabilities, capability{ + donCapabilityConfig: defaultCapabilityRequestConfig, + registryConfig: registryConfig, + }) +} + // TODO - add configuration for remote support - do this for each capability as an option func (d *DON) AddTargetCapability(targetFactory TargetFactory) { d.targetFactories = append(d.targetFactories, targetFactory) } +func (d *DON) AddTriggerCapability(triggerFactory TriggerFactory) { + d.triggerFactories = append(d.triggerFactories, triggerFactory) +} + func (d *DON) AddExternalTriggerCapability(triggerFactory TriggerFactory) { d.triggerFactories = append(d.triggerFactories, triggerFactory) @@ -243,7 +335,7 @@ func (d *DON) AddExternalTriggerCapability(triggerFactory TriggerFactory) { }, } - d.externalCapabilities = append(d.externalCapabilities, triggerCapability) + d.publishedCapabilities = append(d.publishedCapabilities, triggerCapability) } func (d *DON) AddJob(ctx context.Context, j *job.Job) error { @@ -323,9 +415,10 @@ func (d *DON) AddOCR3NonStandardCapability() { CapabilityType: uint8(registrysyncer.ContractCapabilityTypeConsensus), } - d.externalCapabilities = append(d.externalCapabilities, capability{ + d.publishedCapabilities = append(d.publishedCapabilities, capability{ donCapabilityConfig: newCapabilityConfig(), registryConfig: ocr, + internalOnly: true, }) } @@ -357,7 +450,7 @@ func (d *DON) AddEthereumWriteTargetNonStandardCapability(forwarderAddr common.A }, } - d.externalCapabilities = append(d.externalCapabilities, capability{ + d.publishedCapabilities = append(d.publishedCapabilities, capability{ donCapabilityConfig: targetCapabilityConfig, registryConfig: writeChain, }) @@ -366,7 +459,7 @@ func (d *DON) AddEthereumWriteTargetNonStandardCapability(forwarderAddr common.A } func addOCR3Capability(ctx context.Context, t *testing.T, lggr logger.Logger, capabilityRegistry *capabilities.Registry, - libocr *MockLibOCR, donF uint8, ocr2KeyBundle ocr2key.KeyBundle) { + libocr *FakeLibOCR, donF uint8, ocr2KeyBundle ocr2key.KeyBundle) { requestTimeout := 10 * time.Minute cfg := ocr3.Config{ Logger: lggr, @@ -394,6 +487,6 @@ func addOCR3Capability(ctx context.Context, t *testing.T, lggr logger.Logger, ca libocr.AddNode(plugin, transmitter, ocr2KeyBundle) } -func Context(tb testing.TB) context.Context { - return testutils.Context(tb) +func Context(tb testing.TB) (ctx context.Context, cancel func()) { + return context.WithCancel(testutils.Context(tb)) } diff --git a/core/capabilities/integration_tests/framework/mock_dispatcher.go b/core/capabilities/integration_tests/framework/fake_dispatcher.go similarity index 61% rename from core/capabilities/integration_tests/framework/mock_dispatcher.go rename to core/capabilities/integration_tests/framework/fake_dispatcher.go index f208933f1f1..b4223b0742c 100644 --- a/core/capabilities/integration_tests/framework/mock_dispatcher.go +++ b/core/capabilities/integration_tests/framework/fake_dispatcher.go @@ -2,6 +2,7 @@ package framework import ( "context" + "encoding/hex" "fmt" "sync" "testing" @@ -12,15 +13,18 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote" remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" + "github.com/smartcontractkit/libocr/ragep2p/types" "google.golang.org/protobuf/proto" ) -// MockRageP2PNetwork backs the dispatchers created for each node in the test and effectively +// FakeRageP2PNetwork backs the dispatchers created for each node in the test and effectively // acts as the rageP2P network layer. -type MockRageP2PNetwork struct { +type FakeRageP2PNetwork struct { services.StateMachine - t *testing.T + t *testing.T + ctx context.Context + readyError error chanBufferSize int stopCh services.StopChan @@ -28,34 +32,57 @@ type MockRageP2PNetwork struct { peerIDToBrokerNode map[p2ptypes.PeerID]*brokerNode + capabilityRegistrations map[CapabilityRegistration]bool + mux sync.Mutex } -func NewMockRageP2PNetwork(t *testing.T, chanBufferSize int) *MockRageP2PNetwork { - return &MockRageP2PNetwork{ - t: t, - stopCh: make(services.StopChan), - chanBufferSize: chanBufferSize, - peerIDToBrokerNode: make(map[p2ptypes.PeerID]*brokerNode), +func NewFakeRageP2PNetwork(ctx context.Context, t *testing.T, chanBufferSize int) *FakeRageP2PNetwork { + network := &FakeRageP2PNetwork{ + t: t, + ctx: ctx, + stopCh: make(services.StopChan), + chanBufferSize: chanBufferSize, + peerIDToBrokerNode: make(map[p2ptypes.PeerID]*brokerNode), + capabilityRegistrations: make(map[CapabilityRegistration]bool), } + + go func() { + <-ctx.Done() + network.SetReadyError(fmt.Errorf("context done")) + }() + + return network } -func (a *MockRageP2PNetwork) Start(ctx context.Context) error { - return a.StartOnce("MockRageP2PNetwork", func() error { +func (a *FakeRageP2PNetwork) Start(ctx context.Context) error { + return a.StartOnce("FakeRageP2PNetwork", func() error { return nil }) } -func (a *MockRageP2PNetwork) Close() error { - return a.StopOnce("MockRageP2PNetwork", func() error { +func (a *FakeRageP2PNetwork) Close() error { + return a.StopOnce("FakeRageP2PNetwork", func() error { close(a.stopCh) a.wg.Wait() return nil }) } +func (a *FakeRageP2PNetwork) Ready() error { + a.mux.Lock() + defer a.mux.Unlock() + return a.readyError +} + +func (a *FakeRageP2PNetwork) SetReadyError(err error) { + a.mux.Lock() + defer a.mux.Unlock() + a.readyError = err +} + // NewDispatcherForNode creates a new dispatcher for a node with the given peer ID. -func (a *MockRageP2PNetwork) NewDispatcherForNode(nodePeerID p2ptypes.PeerID) remotetypes.Dispatcher { +func (a *FakeRageP2PNetwork) NewDispatcherForNode(nodePeerID p2ptypes.PeerID) remotetypes.Dispatcher { return &brokerDispatcher{ callerPeerID: nodePeerID, broker: a, @@ -63,18 +90,41 @@ func (a *MockRageP2PNetwork) NewDispatcherForNode(nodePeerID p2ptypes.PeerID) re } } -func (a *MockRageP2PNetwork) HealthReport() map[string]error { +func (a *FakeRageP2PNetwork) HealthReport() map[string]error { return nil } -func (a *MockRageP2PNetwork) Name() string { - return "MockRageP2PNetwork" +func (a *FakeRageP2PNetwork) Name() string { + return "FakeRageP2PNetwork" +} + +type CapabilityRegistration struct { + nodePeerID string + capabilityID string + capabilityDonID uint32 } -func (a *MockRageP2PNetwork) registerReceiverNode(nodePeerID p2ptypes.PeerID, capabilityID string, capabilityDonID uint32, receiver remotetypes.Receiver) { +func (a *FakeRageP2PNetwork) GetCapabilityRegistrations() map[CapabilityRegistration]bool { a.mux.Lock() defer a.mux.Unlock() + copiedRegistrations := make(map[CapabilityRegistration]bool) + for k, v := range a.capabilityRegistrations { + copiedRegistrations[k] = v + } + return copiedRegistrations +} + +func (a *FakeRageP2PNetwork) registerReceiverNode(nodePeerID p2ptypes.PeerID, capabilityID string, capabilityDonID uint32, receiver remotetypes.Receiver) { + a.mux.Lock() + defer a.mux.Unlock() + + a.capabilityRegistrations[CapabilityRegistration{ + nodePeerID: hex.EncodeToString(nodePeerID[:]), + capabilityID: capabilityID, + capabilityDonID: capabilityDonID, + }] = true + node, ok := a.peerIDToBrokerNode[nodePeerID] if !ok { node = a.newNode() @@ -90,9 +140,10 @@ func (a *MockRageP2PNetwork) registerReceiverNode(nodePeerID p2ptypes.PeerID, ca } } -func (a *MockRageP2PNetwork) Send(msg *remotetypes.MessageBody) { +func (a *FakeRageP2PNetwork) Send(msg *remotetypes.MessageBody) { peerID := toPeerID(msg.Receiver) - node, ok := a.peerIDToBrokerNode[peerID] + + node, ok := a.getNodeForPeerID(peerID) if !ok { panic(fmt.Sprintf("node not found for peer ID %v", peerID)) } @@ -100,6 +151,13 @@ func (a *MockRageP2PNetwork) Send(msg *remotetypes.MessageBody) { node.receiveCh <- msg } +func (a *FakeRageP2PNetwork) getNodeForPeerID(peerID types.PeerID) (*brokerNode, bool) { + a.mux.Lock() + defer a.mux.Unlock() + node, ok := a.peerIDToBrokerNode[peerID] + return node, ok +} + type brokerNode struct { registerReceiverCh chan *registerReceiverRequest receiveCh chan *remotetypes.MessageBody @@ -115,7 +173,7 @@ type registerReceiverRequest struct { receiver remotetypes.Receiver } -func (a *MockRageP2PNetwork) newNode() *brokerNode { +func (a *FakeRageP2PNetwork) newNode() *brokerNode { n := &brokerNode{ receiveCh: make(chan *remotetypes.MessageBody, a.chanBufferSize), registerReceiverCh: make(chan *registerReceiverRequest, a.chanBufferSize), @@ -155,6 +213,7 @@ func toPeerID(id []byte) p2ptypes.PeerID { type broker interface { Send(msg *remotetypes.MessageBody) + Ready() error } type brokerDispatcher struct { @@ -190,7 +249,7 @@ func (t *brokerDispatcher) SetReceiver(capabilityId string, donId uint32, receiv } t.receivers[k] = receiver - t.broker.(*MockRageP2PNetwork).registerReceiverNode(t.callerPeerID, capabilityId, donId, receiver) + t.broker.(*FakeRageP2PNetwork).registerReceiverNode(t.callerPeerID, capabilityId, donId, receiver) return nil } func (t *brokerDispatcher) RemoveReceiver(capabilityId string, donId uint32) {} @@ -202,7 +261,7 @@ func (t *brokerDispatcher) Close() error { } func (t *brokerDispatcher) Ready() error { - return nil + return t.broker.Ready() } func (t *brokerDispatcher) HealthReport() map[string]error { @@ -210,5 +269,5 @@ func (t *brokerDispatcher) HealthReport() map[string]error { } func (t *brokerDispatcher) Name() string { - return "mockDispatcher" + return "fakeDispatcher" } diff --git a/core/capabilities/integration_tests/framework/mock_libocr.go b/core/capabilities/integration_tests/framework/fake_libocr.go similarity index 64% rename from core/capabilities/integration_tests/framework/mock_libocr.go rename to core/capabilities/integration_tests/framework/fake_libocr.go index 39705031f55..c53186d9685 100644 --- a/core/capabilities/integration_tests/framework/mock_libocr.go +++ b/core/capabilities/integration_tests/framework/fake_libocr.go @@ -24,56 +24,107 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" ) +type oracleContext struct { + ctx context.Context + t *testing.T + lggr logger.Logger + key ocr2key.KeyBundle + N int + F int + protocolRoundInterval time.Duration + mux sync.Mutex + pluginNameToFakeOcr map[string]*FakeLibOCR +} + +func (m *oracleContext) addPlugin(info ocr3types.ReportingPluginInfo, plugin ocr3types.ReportingPlugin[[]byte], + args coretypes.OracleArgs) error { + m.mux.Lock() + defer m.mux.Unlock() + + libOcr := m.pluginNameToFakeOcr[info.Name] + if libOcr == nil { + libOcr = NewFakeLibOCR(m.t, m.lggr, uint8(m.F), m.protocolRoundInterval) + m.pluginNameToFakeOcr[info.Name] = libOcr + } + + libOcr.AddNode(plugin, args.ContractTransmitter, m.key) + + if libOcr.GetNodeCount() == m.N { + err := libOcr.Start(m.ctx) + if err != nil { + return fmt.Errorf("failed to start fake lib ocr: %w", err) + } + } + return nil +} + +func (m *oracleContext) Close() error { + m.mux.Lock() + defer m.mux.Unlock() + + for _, libOcr := range m.pluginNameToFakeOcr { + if err := libOcr.Close(); err != nil { + return fmt.Errorf("failed to close fake lib ocr: %w", err) + } + } + return nil +} + type oracleFactoryFactory struct { - mockLibOCr *MockLibOCR - key ocr2key.KeyBundle - N int - F int + oracleContext *oracleContext } -func newMockLibOcrOracleFactory(mockLibOCr *MockLibOCR, key ocr2key.KeyBundle, N int, F int) *oracleFactoryFactory { +func newFakeOracleFactoryFactory(ctx context.Context, t *testing.T, lggr logger.Logger, key ocr2key.KeyBundle, N int, F int, protocolRoundInterval time.Duration) *oracleFactoryFactory { return &oracleFactoryFactory{ - mockLibOCr: mockLibOCr, - key: key, - N: N, - F: F, + oracleContext: &oracleContext{ + ctx: ctx, + t: t, + lggr: lggr, + key: key, + N: N, + F: F, + protocolRoundInterval: protocolRoundInterval, + pluginNameToFakeOcr: make(map[string]*FakeLibOCR), + }, } } func (o *oracleFactoryFactory) NewOracleFactory(params generic.OracleFactoryParams) (coretypes.OracleFactory, error) { - return &mockOracleFactory{o}, nil + return &fakeOracleFactory{o.oracleContext}, nil } -type mockOracle struct { - *mockOracleFactory - args coretypes.OracleArgs - libocrNodeID string +type fakeOracleFactory struct { + oracleContext *oracleContext } -func (m *mockOracle) Start(ctx context.Context) error { - plugin, _, err := m.args.ReportingPluginFactoryService.NewReportingPlugin(ctx, ocr3types.ReportingPluginConfig{ - F: m.F, - N: m.N, +func (m *fakeOracleFactory) NewOracle(ctx context.Context, args coretypes.OracleArgs) (coretypes.Oracle, error) { + return &fakeOracle{oracleContext: m.oracleContext, args: args}, nil +} + +type fakeOracle struct { + oracleContext *oracleContext + args coretypes.OracleArgs +} + +func (m *fakeOracle) Start(_ context.Context) error { + plugin, info, err := m.args.ReportingPluginFactoryService.NewReportingPlugin(m.oracleContext.ctx, ocr3types.ReportingPluginConfig{ + F: m.oracleContext.F, + N: m.oracleContext.N, }) + if err != nil { return fmt.Errorf("failed to create reporting plugin: %w", err) } - m.libocrNodeID = m.mockLibOCr.AddNode(plugin, m.args.ContractTransmitter, m.key) - return nil -} + if err = m.oracleContext.addPlugin(info, plugin, m.args); err != nil { + return fmt.Errorf("failed to add plugin: %w", err) + } -func (m *mockOracle) Close(ctx context.Context) error { - m.mockLibOCr.RemoveNode(m.libocrNodeID) return nil } -type mockOracleFactory struct { - *oracleFactoryFactory -} - -func (m *mockOracleFactory) NewOracle(ctx context.Context, args coretypes.OracleArgs) (coretypes.Oracle, error) { - return &mockOracle{mockOracleFactory: m, args: args}, nil +func (m *fakeOracle) Close(ctx context.Context) error { + return m.oracleContext.Close() } type libocrNode struct { @@ -83,9 +134,9 @@ type libocrNode struct { key ocr2key.KeyBundle } -// MockLibOCR is a mock libocr implementation for testing purposes that simulates libocr protocol rounds without having +// FakeLibOCR is a fake libocr implementation for testing purposes that simulates libocr protocol rounds without having // to setup the libocr network -type MockLibOCR struct { +type FakeLibOCR struct { services.StateMachine t *testing.T lggr logger.Logger @@ -102,8 +153,8 @@ type MockLibOCR struct { wg sync.WaitGroup } -func NewMockLibOCR(t *testing.T, lggr logger.Logger, f uint8, protocolRoundInterval time.Duration) *MockLibOCR { - return &MockLibOCR{ +func NewFakeLibOCR(t *testing.T, lggr logger.Logger, f uint8, protocolRoundInterval time.Duration) *FakeLibOCR { + return &FakeLibOCR{ t: t, lggr: lggr, f: f, outcomeCtx: ocr3types.OutcomeContext{ @@ -117,8 +168,8 @@ func NewMockLibOCR(t *testing.T, lggr logger.Logger, f uint8, protocolRoundInter } } -func (m *MockLibOCR) Start(ctx context.Context) error { - return m.StartOnce("MockLibOCR", func() error { +func (m *FakeLibOCR) Start(ctx context.Context) error { + return m.StartOnce("FakeLibOCR", func() error { m.wg.Add(1) go func() { defer m.wg.Done() @@ -144,15 +195,15 @@ func (m *MockLibOCR) Start(ctx context.Context) error { }) } -func (m *MockLibOCR) Close() error { - return m.StopOnce("MockLibOCR", func() error { +func (m *FakeLibOCR) Close() error { + return m.StopOnce("FakeLibOCR", func() error { close(m.stopCh) m.wg.Wait() return nil }) } -func (m *MockLibOCR) AddNode(plugin ocr3types.ReportingPlugin[[]byte], transmitter ocr3types.ContractTransmitter[[]byte], key ocr2key.KeyBundle) string { +func (m *FakeLibOCR) AddNode(plugin ocr3types.ReportingPlugin[[]byte], transmitter ocr3types.ContractTransmitter[[]byte], key ocr2key.KeyBundle) string { m.mux.Lock() defer m.mux.Unlock() node := &libocrNode{uuid.New().String(), plugin, transmitter, key} @@ -160,7 +211,13 @@ func (m *MockLibOCR) AddNode(plugin ocr3types.ReportingPlugin[[]byte], transmitt return node.id } -func (m *MockLibOCR) RemoveNode(id string) { +func (m *FakeLibOCR) GetNodeCount() int { + m.mux.Lock() + defer m.mux.Unlock() + return len(m.nodes) +} + +func (m *FakeLibOCR) RemoveNode(id string) { m.mux.Lock() defer m.mux.Unlock() @@ -174,7 +231,7 @@ func (m *MockLibOCR) RemoveNode(id string) { m.nodes = updatedNodes } -func (m *MockLibOCR) simulateProtocolRound(ctx context.Context) error { +func (m *FakeLibOCR) simulateProtocolRound(ctx context.Context) error { m.mux.Lock() defer m.mux.Unlock() if len(m.nodes) == 0 { diff --git a/core/capabilities/integration_tests/framework/mock_target.go b/core/capabilities/integration_tests/framework/fake_target.go similarity index 80% rename from core/capabilities/integration_tests/framework/mock_target.go rename to core/capabilities/integration_tests/framework/fake_target.go index e9c03deaca2..442d35e3595 100644 --- a/core/capabilities/integration_tests/framework/mock_target.go +++ b/core/capabilities/integration_tests/framework/fake_target.go @@ -9,7 +9,7 @@ import ( ) var ( - _ capabilities.ActionCapability = &mockTarget{} + _ capabilities.ActionCapability = &fakeTarget{} ) type TargetSink struct { @@ -18,7 +18,7 @@ type TargetSink struct { targetName string version string - targets []mockTarget + targets []fakeTarget Sink chan capabilities.CapabilityRequest } @@ -56,7 +56,7 @@ func (ts *TargetSink) Close() error { } func (ts *TargetSink) CreateNewTarget(t *testing.T) capabilities.TargetCapability { - target := mockTarget{ + target := fakeTarget{ t: t, targetID: ts.targetID, ch: ts.Sink, @@ -65,29 +65,29 @@ func (ts *TargetSink) CreateNewTarget(t *testing.T) capabilities.TargetCapabilit return &target } -type mockTarget struct { +type fakeTarget struct { t *testing.T targetID string ch chan capabilities.CapabilityRequest } -func (mt *mockTarget) Execute(ctx context.Context, rawRequest capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) { +func (mt *fakeTarget) Execute(ctx context.Context, rawRequest capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) { mt.ch <- rawRequest return capabilities.CapabilityResponse{}, nil } -func (mt *mockTarget) Info(ctx context.Context) (capabilities.CapabilityInfo, error) { +func (mt *fakeTarget) Info(ctx context.Context) (capabilities.CapabilityInfo, error) { return capabilities.MustNewCapabilityInfo( mt.targetID, capabilities.CapabilityTypeTarget, - "mock target for target ID "+mt.targetID, + "fake target for target ID "+mt.targetID, ), nil } -func (mt *mockTarget) RegisterToWorkflow(ctx context.Context, request capabilities.RegisterToWorkflowRequest) error { +func (mt *fakeTarget) RegisterToWorkflow(ctx context.Context, request capabilities.RegisterToWorkflowRequest) error { return nil } -func (mt *mockTarget) UnregisterFromWorkflow(ctx context.Context, request capabilities.UnregisterFromWorkflowRequest) error { +func (mt *fakeTarget) UnregisterFromWorkflow(ctx context.Context, request capabilities.UnregisterFromWorkflowRequest) error { return nil } diff --git a/core/capabilities/integration_tests/framework/mock_trigger.go b/core/capabilities/integration_tests/framework/fake_trigger.go similarity index 84% rename from core/capabilities/integration_tests/framework/mock_trigger.go rename to core/capabilities/integration_tests/framework/fake_trigger.go index afc874af6c3..4274eddf5ca 100644 --- a/core/capabilities/integration_tests/framework/mock_trigger.go +++ b/core/capabilities/integration_tests/framework/fake_trigger.go @@ -20,7 +20,7 @@ type TriggerSink struct { triggerName string version string - triggers []mockTrigger + triggers []fakeTrigger stopCh services.StopChan wg sync.WaitGroup @@ -80,12 +80,12 @@ func (r *TriggerSink) SendOutput(outputs *values.Map) { } func (r *TriggerSink) CreateNewTrigger(t *testing.T) capabilities.TriggerCapability { - trigger := newMockTrigger(t, r.triggerID, &r.wg, r.stopCh) + trigger := newFakeTrigger(t, r.triggerID, &r.wg, r.stopCh) r.triggers = append(r.triggers, trigger) return &trigger } -type mockTrigger struct { +type fakeTrigger struct { t *testing.T triggerID string cancel context.CancelFunc @@ -95,8 +95,8 @@ type mockTrigger struct { stopCh services.StopChan } -func newMockTrigger(t *testing.T, triggerID string, wg *sync.WaitGroup, stopCh services.StopChan) mockTrigger { - return mockTrigger{ +func newFakeTrigger(t *testing.T, triggerID string, wg *sync.WaitGroup, stopCh services.StopChan) fakeTrigger { + return fakeTrigger{ t: t, triggerID: triggerID, toSend: make(chan capabilities.TriggerResponse, 1000), @@ -105,19 +105,19 @@ func newMockTrigger(t *testing.T, triggerID string, wg *sync.WaitGroup, stopCh s } } -func (s *mockTrigger) sendResponse(resp capabilities.TriggerResponse) { +func (s *fakeTrigger) sendResponse(resp capabilities.TriggerResponse) { s.toSend <- resp } -func (s *mockTrigger) Info(ctx context.Context) (capabilities.CapabilityInfo, error) { +func (s *fakeTrigger) Info(ctx context.Context) (capabilities.CapabilityInfo, error) { return capabilities.MustNewCapabilityInfo( s.triggerID, capabilities.CapabilityTypeTrigger, - "mock trigger for trigger id "+s.triggerID, + "fake trigger for trigger id "+s.triggerID, ), nil } -func (s *mockTrigger) RegisterTrigger(ctx context.Context, request capabilities.TriggerRegistrationRequest) (<-chan capabilities.TriggerResponse, error) { +func (s *fakeTrigger) RegisterTrigger(ctx context.Context, request capabilities.TriggerRegistrationRequest) (<-chan capabilities.TriggerResponse, error) { if s.cancel != nil { s.t.Fatal("trigger already registered") } @@ -144,7 +144,7 @@ func (s *mockTrigger) RegisterTrigger(ctx context.Context, request capabilities. return responseCh, nil } -func (s *mockTrigger) UnregisterTrigger(ctx context.Context, request capabilities.TriggerRegistrationRequest) error { +func (s *fakeTrigger) UnregisterTrigger(ctx context.Context, request capabilities.TriggerRegistrationRequest) error { if s.cancel == nil { s.t.Fatal("trigger not registered") } diff --git a/core/capabilities/integration_tests/keystone/keystone_test.go b/core/capabilities/integration_tests/keystone/keystone_test.go index 033bb8a2c76..17bfde7cda9 100644 --- a/core/capabilities/integration_tests/keystone/keystone_test.go +++ b/core/capabilities/integration_tests/keystone/keystone_test.go @@ -17,7 +17,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/values" "github.com/smartcontractkit/chainlink/v2/core/capabilities/integration_tests/framework" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/feeds_consumer" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/logger" reporttypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/types" ) @@ -31,7 +30,9 @@ func Test_OneAtATimeTransmissionSchedule(t *testing.T) { } func testTransmissionSchedule(t *testing.T, deltaStage string, schedule string) { - ctx := testutils.Context(t) + ctx, cancel := framework.Context(t) + defer cancel() + lggr := logger.TestLogger(t) lggr.SetLogLevel(zapcore.InfoLevel) @@ -107,7 +108,7 @@ func waitForConsumerReports(ctx context.Context, t *testing.T, consumer *feeds_c for { select { case <-ctxWithTimeout.Done(): - t.Fatalf("timed out waiting for feed reports, expected %d, received %d", len(triggerFeedReports), feedCount) + t.Fatalf("timed out waiting for feeds reports, expected %d, received %d", len(triggerFeedReports), feedCount) case err := <-feedsSub.Err(): require.NoError(t, err) case feed := <-feedsReceived: diff --git a/core/capabilities/integration_tests/keystone/setup.go b/core/capabilities/integration_tests/keystone/setup.go index f90b582d0ee..a639213ded6 100644 --- a/core/capabilities/integration_tests/keystone/setup.go +++ b/core/capabilities/integration_tests/keystone/setup.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" @@ -46,9 +47,11 @@ func setupKeystoneDons(ctx context.Context, t *testing.T, lggr logger.SugaredLog triggerDon := createKeystoneTriggerDon(ctx, t, lggr, triggerDonInfo, donContext, trigger) - workflowDon.Start(ctx, t) - triggerDon.Start(ctx, t) - writeTargetDon.Start(ctx, t) + servicetest.Run(t, workflowDon) + servicetest.Run(t, triggerDon) + servicetest.Run(t, writeTargetDon) + + donContext.WaitForCapabilitiesToBeExposed(t, workflowDon, triggerDon, writeTargetDon) return workflowDon, consumer } diff --git a/core/capabilities/launcher.go b/core/capabilities/launcher.go index 1d309816c1c..1ed4a6ccb74 100644 --- a/core/capabilities/launcher.go +++ b/core/capabilities/launcher.go @@ -19,7 +19,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/values" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable" remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" "github.com/smartcontractkit/chainlink/v2/core/capabilities/streams" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -293,12 +293,26 @@ func (w *launcher) addRemoteCapabilities(ctx context.Context, myDON registrysync return fmt.Errorf("failed to add trigger shim: %w", err) } case capabilities.CapabilityTypeAction: - w.lggr.Warn("no remote client configured for capability type action, skipping configuration") + newActionFn := func(info capabilities.CapabilityInfo) (capabilityService, error) { + client := executable.NewClient( + info, + myDON.DON, + w.dispatcher, + defaultTargetRequestTimeout, + w.lggr, + ) + return client, nil + } + + err := w.addToRegistryAndSetDispatcher(ctx, capability, remoteDON, newActionFn) + if err != nil { + return fmt.Errorf("failed to add action shim: %w", err) + } case capabilities.CapabilityTypeConsensus: w.lggr.Warn("no remote client configured for capability type consensus, skipping configuration") case capabilities.CapabilityTypeTarget: newTargetFn := func(info capabilities.CapabilityInfo) (capabilityService, error) { - client := target.NewClient( + client := executable.NewClient( info, myDON.DON, w.dispatcher, @@ -412,13 +426,40 @@ func (w *launcher) exposeCapabilities(ctx context.Context, myPeerID p2ptypes.Pee return fmt.Errorf("failed to add server-side receiver: %w", err) } case capabilities.CapabilityTypeAction: - w.lggr.Warn("no remote client configured for capability type action, skipping configuration") + newActionServer := func(capability capabilities.BaseCapability, info capabilities.CapabilityInfo) (remotetypes.ReceiverService, error) { + remoteConfig := &capabilities.RemoteExecutableConfig{} + if capabilityConfig.RemoteTargetConfig != nil { + remoteConfig.RequestHashExcludedAttributes = capabilityConfig.RemoteTargetConfig.RequestHashExcludedAttributes + } + + return executable.NewServer( + capabilityConfig.RemoteExecutableConfig, + myPeerID, + capability.(capabilities.ActionCapability), + info, + don.DON, + idsToDONs, + w.dispatcher, + defaultTargetRequestTimeout, + w.lggr, + ), nil + } + + err = w.addReceiver(ctx, capability, don, newActionServer) + if err != nil { + return fmt.Errorf("failed to add action server-side receiver: %w", err) + } case capabilities.CapabilityTypeConsensus: w.lggr.Warn("no remote client configured for capability type consensus, skipping configuration") case capabilities.CapabilityTypeTarget: newTargetServer := func(capability capabilities.BaseCapability, info capabilities.CapabilityInfo) (remotetypes.ReceiverService, error) { - return target.NewServer( - capabilityConfig.RemoteTargetConfig, + remoteConfig := &capabilities.RemoteExecutableConfig{} + if capabilityConfig.RemoteTargetConfig != nil { + remoteConfig.RequestHashExcludedAttributes = capabilityConfig.RemoteTargetConfig.RequestHashExcludedAttributes + } + + return executable.NewServer( + remoteConfig, myPeerID, capability.(capabilities.TargetCapability), info, diff --git a/core/capabilities/launcher_test.go b/core/capabilities/launcher_test.go index 11482b0dd30..dc71150ee32 100644 --- a/core/capabilities/launcher_test.go +++ b/core/capabilities/launcher_test.go @@ -185,7 +185,7 @@ func TestLauncher_WiresUpExternalCapabilities(t *testing.T) { ) dispatcher.On("SetReceiver", fullTriggerCapID, dID, mock.AnythingOfType("*remote.triggerPublisher")).Return(nil) - dispatcher.On("SetReceiver", fullTargetID, dID, mock.AnythingOfType("*target.server")).Return(nil) + dispatcher.On("SetReceiver", fullTargetID, dID, mock.AnythingOfType("*executable.server")).Return(nil) err = launcher.Launch(ctx, state) require.NoError(t, err) @@ -376,7 +376,8 @@ func TestLauncher_RemoteTriggerModeAggregatorShim(t *testing.T) { ) dispatcher.On("SetReceiver", fullTriggerCapID, capDonID, mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil) - dispatcher.On("SetReceiver", fullTargetID, capDonID, mock.AnythingOfType("*target.client")).Return(nil) + dispatcher.On("SetReceiver", fullTargetID, capDonID, mock.AnythingOfType("*executable.client")).Return(nil) + dispatcher.On("Ready").Return(nil) awaitRegistrationMessageCh := make(chan struct{}) dispatcher.On("Send", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { select { @@ -692,7 +693,7 @@ func TestLauncher_WiresUpClientsForPublicWorkflowDON(t *testing.T) { ) dispatcher.On("SetReceiver", fullTriggerCapID, capDonID, mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil) - dispatcher.On("SetReceiver", fullTargetID, capDonID, mock.AnythingOfType("*target.client")).Return(nil) + dispatcher.On("SetReceiver", fullTargetID, capDonID, mock.AnythingOfType("*executable.client")).Return(nil) err = launcher.Launch(ctx, state) require.NoError(t, err) diff --git a/core/capabilities/remote/target/client.go b/core/capabilities/remote/executable/client.go similarity index 51% rename from core/capabilities/remote/target/client.go rename to core/capabilities/remote/executable/client.go index 64dcbd14f01..187a62c77e1 100644 --- a/core/capabilities/remote/target/client.go +++ b/core/capabilities/remote/executable/client.go @@ -1,4 +1,4 @@ -package target +package executable import ( "context" @@ -8,15 +8,15 @@ import ( "time" commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target/request" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable/request" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/validation" "github.com/smartcontractkit/chainlink/v2/core/logger" ) -// client is a shim for remote target capabilities. +// client is a shim for remote executable capabilities. // It translates between capability API calls and network messages. // Its responsibilities are: // 1. Transmit capability requests to remote nodes according to a transmission schedule @@ -31,25 +31,25 @@ type client struct { dispatcher types.Dispatcher requestTimeout time.Duration - messageIDToCallerRequest map[string]*request.ClientRequest + requestIDToCallerRequest map[string]*request.ClientRequest mutex sync.Mutex stopCh services.StopChan wg sync.WaitGroup } -var _ commoncap.TargetCapability = &client{} +var _ commoncap.ExecutableCapability = &client{} var _ types.Receiver = &client{} var _ services.Service = &client{} func NewClient(remoteCapabilityInfo commoncap.CapabilityInfo, localDonInfo commoncap.DON, dispatcher types.Dispatcher, requestTimeout time.Duration, lggr logger.Logger) *client { return &client{ - lggr: lggr.Named("TargetClient"), + lggr: lggr.Named("ExecutableCapabilityClient"), remoteCapabilityInfo: remoteCapabilityInfo, localDONInfo: localDonInfo, dispatcher: dispatcher, requestTimeout: requestTimeout, - messageIDToCallerRequest: make(map[string]*request.ClientRequest), + requestIDToCallerRequest: make(map[string]*request.ClientRequest), stopCh: make(services.StopChan), } } @@ -61,7 +61,13 @@ func (c *client) Start(ctx context.Context) error { defer c.wg.Done() c.checkForExpiredRequests() }() - c.lggr.Info("TargetClient started") + c.wg.Add(1) + go func() { + defer c.wg.Done() + c.checkDispatcherReady() + }() + + c.lggr.Info("ExecutableCapability Client started") return nil }) } @@ -71,11 +77,26 @@ func (c *client) Close() error { close(c.stopCh) c.cancelAllRequests(errors.New("client closed")) c.wg.Wait() - c.lggr.Info("TargetClient closed") + c.lggr.Info("ExecutableCapability closed") return nil }) } +func (c *client) checkDispatcherReady() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-c.stopCh: + return + case <-ticker.C: + if err := c.dispatcher.Ready(); err != nil { + c.cancelAllRequests(fmt.Errorf("dispatcher not ready: %w", err)) + } + } + } +} + func (c *client) checkForExpiredRequests() { ticker := time.NewTicker(c.requestTimeout) defer ticker.Stop() @@ -93,10 +114,15 @@ func (c *client) expireRequests() { c.mutex.Lock() defer c.mutex.Unlock() - for messageID, req := range c.messageIDToCallerRequest { + for messageID, req := range c.requestIDToCallerRequest { if req.Expired() { req.Cancel(errors.New("request expired")) - delete(c.messageIDToCallerRequest, messageID) + delete(c.requestIDToCallerRequest, messageID) + } + + if c.dispatcher.Ready() != nil { + c.cancelAllRequests(errors.New("dispatcher not ready")) + return } } } @@ -104,7 +130,7 @@ func (c *client) expireRequests() { func (c *client) cancelAllRequests(err error) { c.mutex.Lock() defer c.mutex.Unlock() - for _, req := range c.messageIDToCallerRequest { + for _, req := range c.requestIDToCallerRequest { req.Cancel(err) } } @@ -113,49 +139,81 @@ func (c *client) Info(ctx context.Context) (commoncap.CapabilityInfo, error) { return c.remoteCapabilityInfo, nil } -func (c *client) RegisterToWorkflow(ctx context.Context, request commoncap.RegisterToWorkflowRequest) error { - // do nothing - return nil -} +func (c *client) RegisterToWorkflow(ctx context.Context, registerRequest commoncap.RegisterToWorkflowRequest) error { -func (c *client) UnregisterFromWorkflow(ctx context.Context, request commoncap.UnregisterFromWorkflowRequest) error { - // do nothing - return nil -} + req, err := request.NewClientRegisterToWorkflowRequest(ctx, c.lggr, registerRequest, c.remoteCapabilityInfo, c.localDONInfo, c.dispatcher, + c.requestTimeout) -func (c *client) Execute(ctx context.Context, capReq commoncap.CapabilityRequest) (commoncap.CapabilityResponse, error) { - req, err := c.executeRequest(ctx, capReq) if err != nil { - return commoncap.CapabilityResponse{}, fmt.Errorf("failed to execute request: %w", err) + return fmt.Errorf("failed to create client request: %w", err) + } + + if err = c.sendRequest(req); err != nil { + return fmt.Errorf("failed to send request: %w", err) } resp := <-req.ResponseChan() - return resp.CapabilityResponse, resp.Err + if resp.Err != nil { + return fmt.Errorf("error executing request: %w", resp.Err) + } + return nil } -func (c *client) executeRequest(ctx context.Context, capReq commoncap.CapabilityRequest) (*request.ClientRequest, error) { - c.mutex.Lock() - defer c.mutex.Unlock() +func (c *client) UnregisterFromWorkflow(ctx context.Context, unregisterRequest commoncap.UnregisterFromWorkflowRequest) error { + req, err := request.NewClientUnregisterFromWorkflowRequest(ctx, c.lggr, unregisterRequest, c.remoteCapabilityInfo, + c.localDONInfo, c.dispatcher, c.requestTimeout) - messageID, err := GetMessageIDForRequest(capReq) if err != nil { - return nil, fmt.Errorf("failed to get message ID for request: %w", err) + return fmt.Errorf("failed to create client request: %w", err) } - c.lggr.Debugw("executing remote target", "messageID", messageID) + if err = c.sendRequest(req); err != nil { + return fmt.Errorf("failed to send request: %w", err) + } - if _, ok := c.messageIDToCallerRequest[messageID]; ok { - return nil, fmt.Errorf("request for message ID %s already exists", messageID) + resp := <-req.ResponseChan() + if resp.Err != nil { + return fmt.Errorf("error executing request: %w", resp.Err) } + return nil +} - req, err := request.NewClientRequest(ctx, c.lggr, capReq, messageID, c.remoteCapabilityInfo, c.localDONInfo, c.dispatcher, +func (c *client) Execute(ctx context.Context, capReq commoncap.CapabilityRequest) (commoncap.CapabilityResponse, error) { + req, err := request.NewClientExecuteRequest(ctx, c.lggr, capReq, c.remoteCapabilityInfo, c.localDONInfo, c.dispatcher, c.requestTimeout) if err != nil { - return nil, fmt.Errorf("failed to create client request: %w", err) + return commoncap.CapabilityResponse{}, fmt.Errorf("failed to create client request: %w", err) + } + + if err = c.sendRequest(req); err != nil { + return commoncap.CapabilityResponse{}, fmt.Errorf("failed to send request: %w", err) } - c.messageIDToCallerRequest[messageID] = req - return req, nil + resp := <-req.ResponseChan() + if resp.Err != nil { + return commoncap.CapabilityResponse{}, fmt.Errorf("error executing request: %w", resp.Err) + } + + capabilityResponse, err := pb.UnmarshalCapabilityResponse(resp.Result) + if err != nil { + return commoncap.CapabilityResponse{}, fmt.Errorf("failed to unmarshal capability response: %w", err) + } + + return capabilityResponse, nil +} + +func (c *client) sendRequest(req *request.ClientRequest) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.lggr.Debugw("executing remote execute capability", "requestID", req.ID()) + + if _, ok := c.requestIDToCallerRequest[req.ID()]; ok { + return fmt.Errorf("request for ID %s already exists", req.ID()) + } + + c.requestIDToCallerRequest[req.ID()] = req + return nil } func (c *client) Receive(ctx context.Context, msg *types.MessageBody) { @@ -168,9 +226,9 @@ func (c *client) Receive(ctx context.Context, msg *types.MessageBody) { return } - c.lggr.Debugw("Remote client target receiving message", "messageID", messageID) + c.lggr.Debugw("Remote client executable receiving message", "messageID", messageID) - req := c.messageIDToCallerRequest[messageID] + req := c.requestIDToCallerRequest[messageID] if req == nil { c.lggr.Warnw("received response for unknown message ID ", "messageID", messageID) return @@ -181,18 +239,6 @@ func (c *client) Receive(ctx context.Context, msg *types.MessageBody) { } } -func GetMessageIDForRequest(req commoncap.CapabilityRequest) (string, error) { - if err := validation.ValidateWorkflowOrExecutionID(req.Metadata.WorkflowID); err != nil { - return "", fmt.Errorf("workflow ID is invalid: %w", err) - } - - if err := validation.ValidateWorkflowOrExecutionID(req.Metadata.WorkflowExecutionID); err != nil { - return "", fmt.Errorf("workflow execution ID is invalid: %w", err) - } - - return req.Metadata.WorkflowID + req.Metadata.WorkflowExecutionID, nil -} - func (c *client) Ready() error { return nil } @@ -202,5 +248,5 @@ func (c *client) HealthReport() map[string]error { } func (c *client) Name() string { - return "TargetClient" + return "ExecutableCapabilityClient" } diff --git a/core/capabilities/remote/executable/client_test.go b/core/capabilities/remote/executable/client_test.go new file mode 100644 index 00000000000..392e15db30b --- /dev/null +++ b/core/capabilities/remote/executable/client_test.go @@ -0,0 +1,385 @@ +package executable_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink-common/pkg/values" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable" + remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/transmission" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/logger" + p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" +) + +const ( + stepReferenceID1 = "step1" + workflowID1 = "15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0" + workflowExecutionID1 = "95ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0abbadeed" +) + +func Test_Client_DonTopologies(t *testing.T) { + ctx := testutils.Context(t) + + transmissionSchedule, err := values.NewMap(map[string]any{ + "schedule": transmission.Schedule_OneAtATime, + "deltaStage": "10ms", + }) + require.NoError(t, err) + + responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { + require.NoError(t, responseError) + mp, err := response.Value.Unwrap() + require.NoError(t, err) + assert.Equal(t, "aValue1", mp.(map[string]any)["response"].(string)) + } + + capability := &TestCapability{} + + responseTimeOut := 10 * time.Minute + + var methods []func(caller commoncap.ExecutableCapability) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + executeInputs, err := values.NewMap(map[string]any{"executeValue1": "aValue1"}) + require.NoError(t, err) + executeMethod(ctx, caller, transmissionSchedule, executeInputs, responseTest, t) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + registerToWorkflowMethod(ctx, caller, transmissionSchedule, func(t *testing.T, responseError error) { + require.NoError(t, responseError) + }, t) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + unregisterFromWorkflowMethod(ctx, caller, transmissionSchedule, func(t *testing.T, responseError error) { + require.NoError(t, responseError) + }, t) + }) + + for _, method := range methods { + + testClient(t, 1, responseTimeOut, 1, 0, + capability, method) + + testClient(t, 10, responseTimeOut, 1, 0, + capability, method) + + testClient(t, 1, responseTimeOut, 10, 3, + capability, method) + + testClient(t, 10, responseTimeOut, 10, 3, + capability, method) + + testClient(t, 10, responseTimeOut, 10, 9, + capability, method) + } +} + +func Test_Client_TransmissionSchedules(t *testing.T) { + ctx := testutils.Context(t) + + responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { + require.NoError(t, responseError) + mp, err := response.Value.Unwrap() + require.NoError(t, err) + assert.Equal(t, "aValue1", mp.(map[string]any)["response"].(string)) + } + + capability := &TestCapability{} + + responseTimeOut := 10 * time.Minute + + transmissionSchedule, err := values.NewMap(map[string]any{ + "schedule": transmission.Schedule_OneAtATime, + "deltaStage": "10ms", + }) + require.NoError(t, err) + + testClient(t, 1, responseTimeOut, 1, 0, + capability, func(caller commoncap.ExecutableCapability) { + executeInputs, err := values.NewMap(map[string]any{"executeValue1": "aValue1"}) + require.NoError(t, err) + executeMethod(ctx, caller, transmissionSchedule, executeInputs, responseTest, t) + }) + testClient(t, 10, responseTimeOut, 10, 3, + capability, func(caller commoncap.ExecutableCapability) { + executeInputs, err := values.NewMap(map[string]any{"executeValue1": "aValue1"}) + require.NoError(t, err) + executeMethod(ctx, caller, transmissionSchedule, executeInputs, responseTest, t) + }) + + transmissionSchedule, err = values.NewMap(map[string]any{ + "schedule": transmission.Schedule_AllAtOnce, + "deltaStage": "10ms", + }) + require.NoError(t, err) + + testClient(t, 1, responseTimeOut, 1, 0, + capability, func(caller commoncap.ExecutableCapability) { + executeInputs, err := values.NewMap(map[string]any{"executeValue1": "aValue1"}) + require.NoError(t, err) + executeMethod(ctx, caller, transmissionSchedule, executeInputs, responseTest, t) + }) + testClient(t, 10, responseTimeOut, 10, 3, + capability, func(caller commoncap.ExecutableCapability) { + executeInputs, err := values.NewMap(map[string]any{"executeValue1": "aValue1"}) + require.NoError(t, err) + executeMethod(ctx, caller, transmissionSchedule, executeInputs, responseTest, t) + }) +} + +func Test_Client_TimesOutIfInsufficientCapabilityPeerResponses(t *testing.T) { + ctx := testutils.Context(t) + + responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { + assert.NotNil(t, responseError) + } + + capability := &TestCapability{} + + transmissionSchedule, err := values.NewMap(map[string]any{ + "schedule": transmission.Schedule_AllAtOnce, + "deltaStage": "10ms", + }) + require.NoError(t, err) + + // number of capability peers is less than F + 1 + + testClient(t, 10, 1*time.Second, 10, 11, + capability, + func(caller commoncap.ExecutableCapability) { + executeInputs, err := values.NewMap(map[string]any{"executeValue1": "aValue1"}) + require.NoError(t, err) + executeMethod(ctx, caller, transmissionSchedule, executeInputs, responseTest, t) + }) +} + +func testClient(t *testing.T, numWorkflowPeers int, workflowNodeResponseTimeout time.Duration, + numCapabilityPeers int, capabilityDonF uint8, underlying commoncap.ExecutableCapability, + method func(caller commoncap.ExecutableCapability)) { + lggr := logger.TestLogger(t) + + capabilityPeers := make([]p2ptypes.PeerID, numCapabilityPeers) + for i := 0; i < numCapabilityPeers; i++ { + capabilityPeers[i] = NewP2PPeerID(t) + } + + capDonInfo := commoncap.DON{ + ID: 1, + Members: capabilityPeers, + F: capabilityDonF, + } + + capInfo := commoncap.CapabilityInfo{ + ID: "cap_id@1.0.0", + CapabilityType: commoncap.CapabilityTypeTrigger, + Description: "Remote Executable Capability", + DON: &capDonInfo, + } + + workflowPeers := make([]p2ptypes.PeerID, numWorkflowPeers) + for i := 0; i < numWorkflowPeers; i++ { + workflowPeers[i] = NewP2PPeerID(t) + } + + workflowDonInfo := commoncap.DON{ + Members: workflowPeers, + ID: 2, + } + + broker := newTestAsyncMessageBroker(t, 100) + + receivers := make([]remotetypes.Receiver, numCapabilityPeers) + for i := 0; i < numCapabilityPeers; i++ { + capabilityDispatcher := broker.NewDispatcherForNode(capabilityPeers[i]) + receiver := newTestServer(capabilityPeers[i], capabilityDispatcher, workflowDonInfo, underlying) + broker.RegisterReceiverNode(capabilityPeers[i], receiver) + receivers[i] = receiver + } + + callers := make([]commoncap.ExecutableCapability, numWorkflowPeers) + + for i := 0; i < numWorkflowPeers; i++ { + workflowPeerDispatcher := broker.NewDispatcherForNode(workflowPeers[i]) + caller := executable.NewClient(capInfo, workflowDonInfo, workflowPeerDispatcher, workflowNodeResponseTimeout, lggr) + servicetest.Run(t, caller) + broker.RegisterReceiverNode(workflowPeers[i], caller) + callers[i] = caller + } + + servicetest.Run(t, broker) + + wg := &sync.WaitGroup{} + wg.Add(len(callers)) + + // Fire off all the requests + for _, caller := range callers { + go func(caller commoncap.ExecutableCapability) { + defer wg.Done() + method(caller) + }(caller) + } + + wg.Wait() +} + +func registerToWorkflowMethod(ctx context.Context, caller commoncap.ExecutableCapability, transmissionSchedule *values.Map, + responseTest func(t *testing.T, responseError error), t *testing.T) { + + err := caller.RegisterToWorkflow(ctx, commoncap.RegisterToWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + ReferenceID: stepReferenceID1, + }, + Config: transmissionSchedule, + }) + + responseTest(t, err) +} + +func unregisterFromWorkflowMethod(ctx context.Context, caller commoncap.ExecutableCapability, transmissionSchedule *values.Map, + responseTest func(t *testing.T, responseError error), t *testing.T) { + + err := caller.UnregisterFromWorkflow(ctx, commoncap.UnregisterFromWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + ReferenceID: stepReferenceID1, + }, + Config: transmissionSchedule, + }) + + responseTest(t, err) +} + +func executeMethod(ctx context.Context, caller commoncap.ExecutableCapability, transmissionSchedule *values.Map, + executeInputs *values.Map, responseTest func(t *testing.T, responseCh commoncap.CapabilityResponse, responseError error), t *testing.T) { + responseCh, err := caller.Execute(ctx, + commoncap.CapabilityRequest{ + Metadata: commoncap.RequestMetadata{ + WorkflowID: workflowID1, + WorkflowExecutionID: workflowExecutionID1, + }, + Config: transmissionSchedule, + Inputs: executeInputs, + }) + + responseTest(t, responseCh, err) +} + +// Simple client that only responds once it has received a message from each workflow peer +type clientTestServer struct { + peerID p2ptypes.PeerID + dispatcher remotetypes.Dispatcher + workflowDonInfo commoncap.DON + messageIDToSenders map[string]map[p2ptypes.PeerID]bool + + executableCapability commoncap.ExecutableCapability + + mux sync.Mutex +} + +func newTestServer(peerID p2ptypes.PeerID, dispatcher remotetypes.Dispatcher, workflowDonInfo commoncap.DON, + executableCapability commoncap.ExecutableCapability) *clientTestServer { + return &clientTestServer{ + dispatcher: dispatcher, + workflowDonInfo: workflowDonInfo, + peerID: peerID, + messageIDToSenders: make(map[string]map[p2ptypes.PeerID]bool), + executableCapability: executableCapability, + } +} + +func (t *clientTestServer) Receive(_ context.Context, msg *remotetypes.MessageBody) { + t.mux.Lock() + defer t.mux.Unlock() + + sender := toPeerID(msg.Sender) + messageID, err := executable.GetMessageID(msg) + if err != nil { + panic(err) + } + + if t.messageIDToSenders[messageID] == nil { + t.messageIDToSenders[messageID] = make(map[p2ptypes.PeerID]bool) + } + + sendersOfMessageID := t.messageIDToSenders[messageID] + if sendersOfMessageID[sender] { + panic("received duplicate message") + } + + sendersOfMessageID[sender] = true + + if len(t.messageIDToSenders[messageID]) == len(t.workflowDonInfo.Members) { + + switch msg.Method { + case remotetypes.MethodExecute: + capabilityRequest, err := pb.UnmarshalCapabilityRequest(msg.Payload) + if err != nil { + panic(err) + } + resp, responseErr := t.executableCapability.Execute(context.Background(), capabilityRequest) + payload, marshalErr := pb.MarshalCapabilityResponse(resp) + t.sendResponse(messageID, responseErr, payload, marshalErr) + + case remotetypes.MethodRegisterToWorkflow: + registerRequest, err := pb.UnmarshalRegisterToWorkflowRequest(msg.Payload) + if err != nil { + panic(err) + } + responseErr := t.executableCapability.RegisterToWorkflow(context.Background(), registerRequest) + t.sendResponse(messageID, responseErr, nil, nil) + case remotetypes.MethodUnregisterFromWorkflow: + unregisterRequest, err := pb.UnmarshalUnregisterFromWorkflowRequest(msg.Payload) + if err != nil { + panic(err) + } + responseErr := t.executableCapability.UnregisterFromWorkflow(context.Background(), unregisterRequest) + t.sendResponse(messageID, responseErr, nil, nil) + default: + panic("unknown method") + } + + } +} + +func (t *clientTestServer) sendResponse(messageID string, responseErr error, + payload []byte, marshalErr error) { + for receiver := range t.messageIDToSenders[messageID] { + var responseMsg = &remotetypes.MessageBody{ + CapabilityId: "cap_id@1.0.0", + CapabilityDonId: 1, + CallerDonId: t.workflowDonInfo.ID, + Method: remotetypes.MethodExecute, + MessageId: []byte(messageID), + Sender: t.peerID[:], + Receiver: receiver[:], + } + + if responseErr != nil { + responseMsg.Error = remotetypes.Error_INTERNAL_ERROR + } else { + + if marshalErr != nil { + panic(marshalErr) + } + responseMsg.Payload = payload + } + + err := t.dispatcher.Send(receiver, responseMsg) + if err != nil { + panic(err) + } + } +} diff --git a/core/capabilities/remote/target/endtoend_test.go b/core/capabilities/remote/executable/endtoend_test.go similarity index 52% rename from core/capabilities/remote/target/endtoend_test.go rename to core/capabilities/remote/executable/endtoend_test.go index 0cade4f7855..d65a24f5c30 100644 --- a/core/capabilities/remote/target/endtoend_test.go +++ b/core/capabilities/remote/executable/endtoend_test.go @@ -1,4 +1,4 @@ -package target_test +package executable_test import ( "context" @@ -18,7 +18,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink-common/pkg/values" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable" remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" "github.com/smartcontractkit/chainlink/v2/core/capabilities/transmission" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" @@ -26,7 +26,7 @@ import ( p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" ) -func Test_RemoteTargetCapability_InsufficientCapabilityResponses(t *testing.T) { +func Test_RemoteExecutableCapability_InsufficientCapabilityResponses(t *testing.T) { ctx := testutils.Context(t) responseTest := func(t *testing.T, responseCh commoncap.CapabilityResponse, responseError error) { @@ -41,10 +41,30 @@ func Test_RemoteTargetCapability_InsufficientCapabilityResponses(t *testing.T) { }) require.NoError(t, err) - testRemoteTarget(ctx, t, capability, 10, 9, 10*time.Millisecond, 10, 10, 10*time.Minute, transmissionSchedule, responseTest) + var methods []func(caller commoncap.ExecutableCapability) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + executeCapability(caller, t, ctx, transmissionSchedule, responseTest) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + registerWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + require.NotNil(t, responseError) + }) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + unregisterWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + require.NotNil(t, responseError) + }) + }) + + for _, method := range methods { + testRemoteExecutableCapability(ctx, t, capability, 10, 9, 10*time.Millisecond, 10, 10, 10*time.Minute, method) + } } -func Test_RemoteTargetCapability_InsufficientWorkflowRequests(t *testing.T) { +func Test_RemoteExecutableCapability_InsufficientWorkflowRequests(t *testing.T) { ctx := testutils.Context(t) responseTest := func(t *testing.T, responseCh commoncap.CapabilityResponse, responseError error) { @@ -61,10 +81,30 @@ func Test_RemoteTargetCapability_InsufficientWorkflowRequests(t *testing.T) { }) require.NoError(t, err) - testRemoteTarget(ctx, t, capability, 10, 10, 10*time.Millisecond, 10, 9, timeOut, transmissionSchedule, responseTest) + var methods []func(caller commoncap.ExecutableCapability) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + executeCapability(caller, t, ctx, transmissionSchedule, responseTest) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + registerWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + require.NotNil(t, responseError) + }) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + unregisterWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + require.NotNil(t, responseError) + }) + }) + + for _, method := range methods { + testRemoteExecutableCapability(ctx, t, capability, 10, 10, 10*time.Millisecond, 10, 9, timeOut, method) + } } -func Test_RemoteTargetCapability_TransmissionSchedules(t *testing.T) { +func Test_RemoteExecutableCapability_TransmissionSchedules(t *testing.T) { ctx := testutils.Context(t) responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { @@ -84,18 +124,24 @@ func Test_RemoteTargetCapability_TransmissionSchedules(t *testing.T) { capability := &TestCapability{} - testRemoteTarget(ctx, t, capability, 10, 9, timeOut, 10, 9, timeOut, transmissionSchedule, responseTest) + method := func(caller commoncap.ExecutableCapability) { + executeCapability(caller, t, ctx, transmissionSchedule, responseTest) + } + testRemoteExecutableCapability(ctx, t, capability, 10, 9, timeOut, 10, 9, timeOut, method) transmissionSchedule, err = values.NewMap(map[string]any{ "schedule": transmission.Schedule_AllAtOnce, "deltaStage": "10ms", }) require.NoError(t, err) + method = func(caller commoncap.ExecutableCapability) { + executeCapability(caller, t, ctx, transmissionSchedule, responseTest) + } - testRemoteTarget(ctx, t, capability, 10, 9, timeOut, 10, 9, timeOut, transmissionSchedule, responseTest) + testRemoteExecutableCapability(ctx, t, capability, 10, 9, timeOut, 10, 9, timeOut, method) } -func Test_RemoteTargetCapability_DonTopologies(t *testing.T) { +func Test_RemoteExecutionCapability_DonTopologies(t *testing.T) { ctx := testutils.Context(t) responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { @@ -115,26 +161,43 @@ func Test_RemoteTargetCapability_DonTopologies(t *testing.T) { capability := &TestCapability{} - // Test scenarios where the number of submissions is greater than or equal to F + 1 - testRemoteTarget(ctx, t, capability, 1, 0, timeOut, 1, 0, timeOut, transmissionSchedule, responseTest) - testRemoteTarget(ctx, t, capability, 4, 3, timeOut, 1, 0, timeOut, transmissionSchedule, responseTest) - testRemoteTarget(ctx, t, capability, 10, 3, timeOut, 1, 0, timeOut, transmissionSchedule, responseTest) + var methods []func(caller commoncap.ExecutableCapability) - testRemoteTarget(ctx, t, capability, 1, 0, timeOut, 1, 0, timeOut, transmissionSchedule, responseTest) - testRemoteTarget(ctx, t, capability, 1, 0, timeOut, 4, 3, timeOut, transmissionSchedule, responseTest) - testRemoteTarget(ctx, t, capability, 1, 0, timeOut, 10, 3, timeOut, transmissionSchedule, responseTest) + methods = append(methods, func(caller commoncap.ExecutableCapability) { + executeCapability(caller, t, ctx, transmissionSchedule, responseTest) + }) - testRemoteTarget(ctx, t, capability, 4, 3, timeOut, 4, 3, timeOut, transmissionSchedule, responseTest) - testRemoteTarget(ctx, t, capability, 10, 3, timeOut, 10, 3, timeOut, transmissionSchedule, responseTest) - testRemoteTarget(ctx, t, capability, 10, 9, timeOut, 10, 9, timeOut, transmissionSchedule, responseTest) -} + methods = append(methods, func(caller commoncap.ExecutableCapability) { + registerWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + require.NoError(t, responseError) + }) + }) -func Test_RemoteTargetCapability_CapabilityError(t *testing.T) { - ctx := testutils.Context(t) + methods = append(methods, func(caller commoncap.ExecutableCapability) { + unregisterWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + require.NoError(t, responseError) + }) + }) - responseTest := func(t *testing.T, responseCh commoncap.CapabilityResponse, responseError error) { - assert.Equal(t, "failed to execute capability: an error", responseError.Error()) + for _, method := range methods { + + // Test scenarios where the number of submissions is greater than or equal to F + 1 + testRemoteExecutableCapability(ctx, t, capability, 1, 0, timeOut, 1, 0, timeOut, method) + testRemoteExecutableCapability(ctx, t, capability, 4, 3, timeOut, 1, 0, timeOut, method) + testRemoteExecutableCapability(ctx, t, capability, 10, 3, timeOut, 1, 0, timeOut, method) + + testRemoteExecutableCapability(ctx, t, capability, 1, 0, timeOut, 1, 0, timeOut, method) + testRemoteExecutableCapability(ctx, t, capability, 1, 0, timeOut, 4, 3, timeOut, method) + testRemoteExecutableCapability(ctx, t, capability, 1, 0, timeOut, 10, 3, timeOut, method) + + testRemoteExecutableCapability(ctx, t, capability, 4, 3, timeOut, 4, 3, timeOut, method) + testRemoteExecutableCapability(ctx, t, capability, 10, 3, timeOut, 10, 3, timeOut, method) + testRemoteExecutableCapability(ctx, t, capability, 10, 9, timeOut, 10, 9, timeOut, method) } +} + +func Test_RemoteExecutionCapability_CapabilityError(t *testing.T) { + ctx := testutils.Context(t) capability := &TestErrorCapability{} @@ -144,15 +207,33 @@ func Test_RemoteTargetCapability_CapabilityError(t *testing.T) { }) require.NoError(t, err) - testRemoteTarget(ctx, t, capability, 10, 9, 10*time.Minute, 10, 9, 10*time.Minute, transmissionSchedule, responseTest) -} + var methods []func(caller commoncap.ExecutableCapability) -func Test_RemoteTargetCapability_RandomCapabilityError(t *testing.T) { - ctx := testutils.Context(t) + methods = append(methods, func(caller commoncap.ExecutableCapability) { + executeCapability(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseCh commoncap.CapabilityResponse, responseError error) { + assert.Equal(t, "error executing request: failed to execute capability: an error", responseError.Error()) + }) + }) - responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { - assert.Equal(t, "request expired", responseError.Error()) + methods = append(methods, func(caller commoncap.ExecutableCapability) { + registerWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + assert.Equal(t, "error executing request: failed to register to workflow: an error", responseError.Error()) + }) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + unregisterWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + assert.Equal(t, "error executing request: failed to unregister from workflow: an error", responseError.Error()) + }) + }) + + for _, method := range methods { + testRemoteExecutableCapability(ctx, t, capability, 10, 9, 10*time.Minute, 10, 9, 10*time.Minute, method) } +} + +func Test_RemoteExecutableCapability_RandomCapabilityError(t *testing.T) { + ctx := testutils.Context(t) capability := &TestRandomErrorCapability{} @@ -162,12 +243,35 @@ func Test_RemoteTargetCapability_RandomCapabilityError(t *testing.T) { }) require.NoError(t, err) - testRemoteTarget(ctx, t, capability, 10, 9, 10*time.Millisecond, 10, 9, 10*time.Minute, transmissionSchedule, responseTest) + var methods []func(caller commoncap.ExecutableCapability) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + executeCapability(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseCh commoncap.CapabilityResponse, responseError error) { + assert.Equal(t, "error executing request: request expired", responseError.Error()) + }) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + registerWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + assert.Equal(t, "error executing request: request expired", responseError.Error()) + }) + }) + + methods = append(methods, func(caller commoncap.ExecutableCapability) { + unregisterWorkflow(caller, t, ctx, transmissionSchedule, func(t *testing.T, responseError error) { + assert.Equal(t, "error executing request: request expired", responseError.Error()) + }) + }) + + for _, method := range methods { + testRemoteExecutableCapability(ctx, t, capability, 10, 9, 10*time.Millisecond, 10, 9, 10*time.Minute, + method) + } } -func testRemoteTarget(ctx context.Context, t *testing.T, underlying commoncap.TargetCapability, numWorkflowPeers int, workflowDonF uint8, workflowNodeTimeout time.Duration, - numCapabilityPeers int, capabilityDonF uint8, capabilityNodeResponseTimeout time.Duration, transmissionSchedule *values.Map, - responseTest func(t *testing.T, response commoncap.CapabilityResponse, responseError error)) { +func testRemoteExecutableCapability(ctx context.Context, t *testing.T, underlying commoncap.ExecutableCapability, numWorkflowPeers int, workflowDonF uint8, workflowNodeTimeout time.Duration, + numCapabilityPeers int, capabilityDonF uint8, capabilityNodeResponseTimeout time.Duration, + method func(caller commoncap.ExecutableCapability)) { lggr := logger.TestLogger(t) capabilityPeers := make([]p2ptypes.PeerID, numCapabilityPeers) @@ -216,17 +320,17 @@ func testRemoteTarget(ctx context.Context, t *testing.T, underlying commoncap.Ta for i := 0; i < numCapabilityPeers; i++ { capabilityPeer := capabilityPeers[i] capabilityDispatcher := broker.NewDispatcherForNode(capabilityPeer) - capabilityNode := target.NewServer(&commoncap.RemoteTargetConfig{RequestHashExcludedAttributes: []string{}}, capabilityPeer, underlying, capInfo, capDonInfo, workflowDONs, capabilityDispatcher, + capabilityNode := executable.NewServer(&commoncap.RemoteExecutableConfig{RequestHashExcludedAttributes: []string{}}, capabilityPeer, underlying, capInfo, capDonInfo, workflowDONs, capabilityDispatcher, capabilityNodeResponseTimeout, lggr) servicetest.Run(t, capabilityNode) broker.RegisterReceiverNode(capabilityPeer, capabilityNode) capabilityNodes[i] = capabilityNode } - workflowNodes := make([]commoncap.TargetCapability, numWorkflowPeers) + workflowNodes := make([]commoncap.ExecutableCapability, numWorkflowPeers) for i := 0; i < numWorkflowPeers; i++ { workflowPeerDispatcher := broker.NewDispatcherForNode(workflowPeers[i]) - workflowNode := target.NewClient(capInfo, workflowDonInfo, workflowPeerDispatcher, workflowNodeTimeout, lggr) + workflowNode := executable.NewClient(capInfo, workflowDonInfo, workflowPeerDispatcher, workflowNodeTimeout, lggr) servicetest.Run(t, workflowNode) broker.RegisterReceiverNode(workflowPeers[i], workflowNode) workflowNodes[i] = workflowNode @@ -234,31 +338,13 @@ func testRemoteTarget(ctx context.Context, t *testing.T, underlying commoncap.Ta servicetest.Run(t, broker) - executeInputs, err := values.NewMap( - map[string]any{ - "executeValue1": "aValue1", - }, - ) - - require.NoError(t, err) - wg := &sync.WaitGroup{} wg.Add(len(workflowNodes)) for _, caller := range workflowNodes { - go func(caller commoncap.TargetCapability) { + go func(caller commoncap.ExecutableCapability) { defer wg.Done() - response, err := caller.Execute(ctx, - commoncap.CapabilityRequest{ - Metadata: commoncap.RequestMetadata{ - WorkflowID: workflowID1, - WorkflowExecutionID: workflowExecutionID1, - }, - Config: transmissionSchedule, - Inputs: executeInputs, - }) - - responseTest(t, response, err) + method(caller) }(caller) } @@ -413,6 +499,14 @@ func (t TestErrorCapability) Execute(ctx context.Context, request commoncap.Capa return commoncap.CapabilityResponse{}, errors.New("an error") } +func (t TestErrorCapability) RegisterToWorkflow(ctx context.Context, request commoncap.RegisterToWorkflowRequest) error { + return errors.New("an error") +} + +func (t TestErrorCapability) UnregisterFromWorkflow(ctx context.Context, request commoncap.UnregisterFromWorkflowRequest) error { + return errors.New("an error") +} + type TestRandomErrorCapability struct { abstractTestCapability } @@ -421,6 +515,14 @@ func (t TestRandomErrorCapability) Execute(ctx context.Context, request commonca return commoncap.CapabilityResponse{}, errors.New(uuid.New().String()) } +func (t TestRandomErrorCapability) RegisterToWorkflow(ctx context.Context, request commoncap.RegisterToWorkflowRequest) error { + return errors.New(uuid.New().String()) +} + +func (t TestRandomErrorCapability) UnregisterFromWorkflow(ctx context.Context, request commoncap.UnregisterFromWorkflowRequest) error { + return errors.New(uuid.New().String()) +} + func NewP2PPeerID(t *testing.T) p2ptypes.PeerID { id := p2ptypes.PeerID{} require.NoError(t, id.UnmarshalText([]byte(NewPeerID()))) @@ -442,3 +544,48 @@ func NewPeerID() string { func libp2pMagic() []byte { return []byte{0x00, 0x24, 0x08, 0x01, 0x12, 0x20} } + +func executeCapability(caller commoncap.ExecutableCapability, t *testing.T, ctx context.Context, transmissionSchedule *values.Map, responseTest func(t *testing.T, response commoncap.CapabilityResponse, responseError error)) { + executeInputs, err := values.NewMap( + map[string]any{ + "executeValue1": "aValue1", + }, + ) + require.NoError(t, err) + response, err := caller.Execute(ctx, + commoncap.CapabilityRequest{ + Metadata: commoncap.RequestMetadata{ + WorkflowID: workflowID1, + WorkflowExecutionID: workflowExecutionID1, + }, + Config: transmissionSchedule, + Inputs: executeInputs, + }) + + responseTest(t, response, err) +} + +func registerWorkflow(caller commoncap.ExecutableCapability, t *testing.T, ctx context.Context, transmissionSchedule *values.Map, responseTest func(t *testing.T, responseError error)) { + + err := caller.RegisterToWorkflow(ctx, commoncap.RegisterToWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + ReferenceID: stepReferenceID1, + }, + Config: transmissionSchedule, + }) + + responseTest(t, err) +} + +func unregisterWorkflow(caller commoncap.ExecutableCapability, t *testing.T, ctx context.Context, transmissionSchedule *values.Map, responseTest func(t *testing.T, responseError error)) { + err := caller.UnregisterFromWorkflow(ctx, commoncap.UnregisterFromWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + ReferenceID: stepReferenceID1, + }, + Config: transmissionSchedule, + }) + + responseTest(t, err) +} diff --git a/core/capabilities/remote/target/request/client_request.go b/core/capabilities/remote/executable/request/client_request.go similarity index 52% rename from core/capabilities/remote/target/request/client_request.go rename to core/capabilities/remote/executable/request/client_request.go index 1a0d707146b..41e1507f172 100644 --- a/core/capabilities/remote/target/request/client_request.go +++ b/core/capabilities/remote/executable/request/client_request.go @@ -10,6 +10,7 @@ import ( "google.golang.org/protobuf/proto" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/validation" ragep2ptypes "github.com/smartcontractkit/libocr/ragep2p/types" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" @@ -22,14 +23,15 @@ import ( p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" ) -type asyncCapabilityResponse struct { - capabilities.CapabilityResponse - Err error +type clientResponse struct { + Result []byte + Err error } type ClientRequest struct { + id string cancelFn context.CancelFunc - responseCh chan asyncCapabilityResponse + responseCh chan clientResponse createdAt time.Time responseIDCount map[[32]byte]int errorCount map[string]int @@ -45,26 +47,95 @@ type ClientRequest struct { wg *sync.WaitGroup } -func NewClientRequest(ctx context.Context, lggr logger.Logger, req commoncap.CapabilityRequest, messageID string, +func NewClientRegisterToWorkflowRequest(ctx context.Context, lggr logger.Logger, req commoncap.RegisterToWorkflowRequest, remoteCapabilityInfo commoncap.CapabilityInfo, localDonInfo capabilities.DON, dispatcher types.Dispatcher, requestTimeout time.Duration) (*ClientRequest, error) { - remoteCapabilityDonInfo := remoteCapabilityInfo.DON - if remoteCapabilityDonInfo == nil { - return nil, errors.New("remote capability info missing DON") + + rawRequest, err := proto.MarshalOptions{Deterministic: true}.Marshal(pb.RegisterToWorkflowRequestToProto(req)) + if err != nil { + return nil, fmt.Errorf("failed to marshal register to workflow request: %w", err) } - rawRequest, err := proto.MarshalOptions{Deterministic: true}.Marshal(pb.CapabilityRequestToProto(req)) + workflowID := req.Metadata.WorkflowID + if err := validation.ValidateWorkflowOrExecutionID(workflowID); err != nil { + return nil, fmt.Errorf("workflow ID is invalid: %w", err) + } + + requestID := types.MethodRegisterToWorkflow + ":" + workflowID + + tc := transmission.TransmissionConfig{ + Schedule: transmission.Schedule_AllAtOnce, + DeltaStage: 0, + } + + return newClientRequest(ctx, lggr, requestID, remoteCapabilityInfo, localDonInfo, dispatcher, requestTimeout, + tc, types.MethodRegisterToWorkflow, rawRequest) +} +func NewClientUnregisterFromWorkflowRequest(ctx context.Context, lggr logger.Logger, req commoncap.UnregisterFromWorkflowRequest, + remoteCapabilityInfo commoncap.CapabilityInfo, localDonInfo capabilities.DON, dispatcher types.Dispatcher, + requestTimeout time.Duration) (*ClientRequest, error) { + + rawRequest, err := proto.MarshalOptions{Deterministic: true}.Marshal(pb.UnregisterFromWorkflowRequestToProto(req)) + if err != nil { + return nil, fmt.Errorf("failed to marshal unregister from workflow request: %w", err) + } + + workflowID := req.Metadata.WorkflowID + if err := validation.ValidateWorkflowOrExecutionID(workflowID); err != nil { + return nil, fmt.Errorf("workflow ID is invalid: %w", err) + } + + requestID := types.MethodUnregisterFromWorkflow + ":" + workflowID + + tc := transmission.TransmissionConfig{ + Schedule: transmission.Schedule_AllAtOnce, + DeltaStage: 0, + } + + return newClientRequest(ctx, lggr, requestID, remoteCapabilityInfo, localDonInfo, dispatcher, requestTimeout, + tc, types.MethodUnregisterFromWorkflow, rawRequest) +} + +func NewClientExecuteRequest(ctx context.Context, lggr logger.Logger, req commoncap.CapabilityRequest, + remoteCapabilityInfo commoncap.CapabilityInfo, localDonInfo capabilities.DON, dispatcher types.Dispatcher, + requestTimeout time.Duration) (*ClientRequest, error) { + + rawRequest, err := proto.MarshalOptions{Deterministic: true}.Marshal(pb.CapabilityRequestToProto(req)) if err != nil { return nil, fmt.Errorf("failed to marshal capability request: %w", err) } - peerIDToTransmissionDelay, err := transmission.GetPeerIDToTransmissionDelay(remoteCapabilityDonInfo.Members, req) + workflowExecutionID := req.Metadata.WorkflowExecutionID + if err := validation.ValidateWorkflowOrExecutionID(workflowExecutionID); err != nil { + return nil, fmt.Errorf("workflow execution ID is invalid: %w", err) + } + + requestID := types.MethodExecute + ":" + workflowExecutionID + + tc, err := transmission.ExtractTransmissionConfig(req.Config) + if err != nil { + return nil, fmt.Errorf("failed to extract transmission config from request: %w", err) + } + + return newClientRequest(ctx, lggr, requestID, remoteCapabilityInfo, localDonInfo, dispatcher, requestTimeout, tc, types.MethodExecute, rawRequest) +} + +func newClientRequest(ctx context.Context, lggr logger.Logger, requestID string, remoteCapabilityInfo commoncap.CapabilityInfo, + localDonInfo commoncap.DON, dispatcher types.Dispatcher, requestTimeout time.Duration, + tc transmission.TransmissionConfig, methodType string, rawRequest []byte) (*ClientRequest, error) { + + remoteCapabilityDonInfo := remoteCapabilityInfo.DON + if remoteCapabilityDonInfo == nil { + return nil, errors.New("remote capability info missing DON") + } + + peerIDToTransmissionDelay, err := transmission.GetPeerIDToTransmissionDelaysForConfig(remoteCapabilityDonInfo.Members, requestID, tc) if err != nil { return nil, fmt.Errorf("failed to get peer ID to transmission delay: %w", err) } - lggr.Debugw("sending request to peers", "execID", req.Metadata.WorkflowExecutionID, "schedule", peerIDToTransmissionDelay) + lggr.Debugw("sending request to peers", "requestID", requestID, "schedule", peerIDToTransmissionDelay) responseReceived := make(map[p2ptypes.PeerID]bool) @@ -79,17 +150,17 @@ func NewClientRequest(ctx context.Context, lggr logger.Logger, req commoncap.Cap CapabilityId: remoteCapabilityInfo.ID, CapabilityDonId: remoteCapabilityDonInfo.ID, CallerDonId: localDonInfo.ID, - Method: types.MethodExecute, + Method: methodType, Payload: rawRequest, - MessageId: []byte(messageID), + MessageId: []byte(requestID), } select { case <-ctxWithCancel.Done(): - lggr.Debugw("context done, not sending request to peer", "execID", req.Metadata.WorkflowExecutionID, "peerID", peerID) + lggr.Debugw("context done, not sending request to peer", "requestID", requestID, "peerID", peerID) return case <-time.After(delay): - lggr.Debugw("sending request to peer", "execID", req.Metadata.WorkflowExecutionID, "peerID", peerID) + lggr.Debugw("sending request to peer", "requestID", requestID, "peerID", peerID) err := dispatcher.Send(peerID, message) if err != nil { lggr.Errorw("failed to send message", "peerID", peerID, "err", err) @@ -99,6 +170,7 @@ func NewClientRequest(ctx context.Context, lggr logger.Logger, req commoncap.Cap } return &ClientRequest{ + id: requestID, cancelFn: cancelFn, createdAt: time.Now(), requestTimeout: requestTimeout, @@ -106,13 +178,17 @@ func NewClientRequest(ctx context.Context, lggr logger.Logger, req commoncap.Cap responseIDCount: make(map[[32]byte]int), errorCount: make(map[string]int), responseReceived: responseReceived, - responseCh: make(chan asyncCapabilityResponse, 1), + responseCh: make(chan clientResponse, 1), wg: wg, lggr: lggr, }, nil } -func (c *ClientRequest) ResponseChan() <-chan asyncCapabilityResponse { +func (c *ClientRequest) ID() string { + return c.id +} + +func (c *ClientRequest) ResponseChan() <-chan clientResponse { return c.responseCh } @@ -126,7 +202,7 @@ func (c *ClientRequest) Cancel(err error) { c.mux.Lock() defer c.mux.Unlock() if !c.respSent { - c.sendResponse(asyncCapabilityResponse{Err: err}) + c.sendResponse(clientResponse{Err: err}) } } @@ -169,24 +245,20 @@ func (c *ClientRequest) OnMessage(_ context.Context, msg *types.MessageBody) err } if c.responseIDCount[responseID] == c.requiredIdenticalResponses { - capabilityResponse, err := pb.UnmarshalCapabilityResponse(msg.Payload) - if err != nil { - c.sendResponse(asyncCapabilityResponse{Err: fmt.Errorf("failed to unmarshal capability response: %w", err)}) - } else { - c.sendResponse(asyncCapabilityResponse{CapabilityResponse: commoncap.CapabilityResponse{Value: capabilityResponse.Value}}) - } + + c.sendResponse(clientResponse{Result: msg.Payload}) } } else { c.lggr.Warnw("received error response", "error", remote.SanitizeLogString(msg.ErrorMsg)) c.errorCount[msg.ErrorMsg]++ if c.errorCount[msg.ErrorMsg] == c.requiredIdenticalResponses { - c.sendResponse(asyncCapabilityResponse{Err: errors.New(msg.ErrorMsg)}) + c.sendResponse(clientResponse{Err: errors.New(msg.ErrorMsg)}) } } return nil } -func (c *ClientRequest) sendResponse(response asyncCapabilityResponse) { +func (c *ClientRequest) sendResponse(response clientResponse) { c.responseCh <- response close(c.responseCh) c.respSent = true diff --git a/core/capabilities/remote/target/request/client_request_test.go b/core/capabilities/remote/executable/request/client_request_test.go similarity index 61% rename from core/capabilities/remote/target/request/client_request_test.go rename to core/capabilities/remote/executable/request/client_request_test.go index 095c73e8ad9..35f2b135ab8 100644 --- a/core/capabilities/remote/target/request/client_request_test.go +++ b/core/capabilities/remote/executable/request/client_request_test.go @@ -12,8 +12,7 @@ import ( commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" "github.com/smartcontractkit/chainlink-common/pkg/values" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target/request" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable/request" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" "github.com/smartcontractkit/chainlink/v2/core/capabilities/transmission" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -80,6 +79,15 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { Config: transmissionSchedule, } + registerToWorkflowRequest := commoncap.RegisterToWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + WorkflowOwner: "0xaa", + ReferenceID: "refID", + }, + Config: transmissionSchedule, + } + m, err := values.NewMap(map[string]any{"response": "response1"}) require.NoError(t, err) capabilityResponse := commoncap.CapabilityResponse{ @@ -89,9 +97,6 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { rawResponse, err := pb.MarshalCapabilityResponse(capabilityResponse) require.NoError(t, err) - messageID, err := target.GetMessageIDForRequest(capabilityRequest) - require.NoError(t, err) - msg := &types.MessageBody{ CapabilityId: capInfo.ID, CapabilityDonId: capDonInfo.ID, @@ -106,7 +111,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { defer cancel() dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} - request, err := request.NewClientRequest(ctx, lggr, capabilityRequest, messageID, capInfo, + request, err := request.NewClientExecuteRequest(ctx, lggr, capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) defer request.Cancel(errors.New("test end")) @@ -149,7 +154,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { defer cancel() dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} - request, err := request.NewClientRequest(ctx, lggr, capabilityRequest, messageID, capInfo, + request, err := request.NewClientExecuteRequest(ctx, lggr, capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) require.NoError(t, err) defer request.Cancel(errors.New("test end")) @@ -175,7 +180,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { defer cancel() dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} - request, err := request.NewClientRequest(ctx, lggr, capabilityRequest, messageID, capInfo, + request, err := request.NewClientExecuteRequest(ctx, lggr, capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) require.NoError(t, err) defer request.Cancel(errors.New("test end")) @@ -198,7 +203,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { defer cancel() dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} - request, err := request.NewClientRequest(ctx, lggr, capabilityRequest, messageID, capInfo, + request, err := request.NewClientExecuteRequest(ctx, lggr, capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) require.NoError(t, err) defer request.Cancel(errors.New("test end")) @@ -236,7 +241,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { defer cancel() dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} - request, err := request.NewClientRequest(ctx, lggr, capabilityRequest, messageID, capInfo, + request, err := request.NewClientExecuteRequest(ctx, lggr, capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) require.NoError(t, err) defer request.Cancel(errors.New("test end")) @@ -281,12 +286,12 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { } }) - t.Run("Send second valid message", func(t *testing.T) { + t.Run("Execute Request", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} - request, err := request.NewClientRequest(ctx, lggr, capabilityRequest, messageID, capInfo, + request, err := request.NewClientExecuteRequest(ctx, lggr, capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) require.NoError(t, err) defer request.Cancel(errors.New("test end")) @@ -304,10 +309,168 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { require.NoError(t, err) response := <-request.ResponseChan() - resp := response.Value.Underlying["response"] + capResponse, err := pb.UnmarshalCapabilityResponse(response.Result) + require.NoError(t, err) + + resp := capResponse.Value.Underlying["response"] assert.Equal(t, resp, values.NewString("response1")) }) + + t.Run("Register To Workflow Request", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + request, err := request.NewClientRegisterToWorkflowRequest(ctx, lggr, registerToWorkflowRequest, capInfo, + workflowDonInfo, dispatcher, 10*time.Minute) + require.NoError(t, err) + defer request.Cancel(errors.New("test end")) + + <-dispatcher.msgs + <-dispatcher.msgs + assert.Equal(t, 0, len(dispatcher.msgs)) + + msg := &types.MessageBody{ + CapabilityId: capInfo.ID, + CapabilityDonId: capDonInfo.ID, + CallerDonId: workflowDonInfo.ID, + Method: types.MethodRegisterToWorkflow, + Payload: nil, + MessageId: []byte("messageID"), + } + + msg.Sender = capabilityPeers[0][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + msg.Sender = capabilityPeers[1][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + response := <-request.ResponseChan() + require.Nil(t, response.Result) + require.NoError(t, response.Err) + }) + + t.Run("Register To Workflow Request with error", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + request, err := request.NewClientRegisterToWorkflowRequest(ctx, lggr, registerToWorkflowRequest, capInfo, + workflowDonInfo, dispatcher, 10*time.Minute) + require.NoError(t, err) + defer request.Cancel(errors.New("test end")) + + <-dispatcher.msgs + <-dispatcher.msgs + assert.Equal(t, 0, len(dispatcher.msgs)) + + msg := &types.MessageBody{ + CapabilityId: capInfo.ID, + CapabilityDonId: capDonInfo.ID, + CallerDonId: workflowDonInfo.ID, + Method: types.MethodRegisterToWorkflow, + Payload: nil, + MessageId: []byte("messageID"), + Error: types.Error_INTERNAL_ERROR, + ErrorMsg: "an error", + } + + msg.Sender = capabilityPeers[0][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + msg.Sender = capabilityPeers[1][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + response := <-request.ResponseChan() + require.Nil(t, response.Result) + assert.Equal(t, "an error", response.Err.Error()) + }) + + t.Run("Unregister From Workflow Request", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + request, err := request.NewClientUnregisterFromWorkflowRequest(ctx, lggr, commoncap.UnregisterFromWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + }, + }, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) + require.NoError(t, err) + defer request.Cancel(errors.New("test end")) + + <-dispatcher.msgs + <-dispatcher.msgs + assert.Equal(t, 0, len(dispatcher.msgs)) + + msg := &types.MessageBody{ + CapabilityId: capInfo.ID, + CapabilityDonId: capDonInfo.ID, + CallerDonId: workflowDonInfo.ID, + Method: types.MethodUnregisterFromWorkflow, + Payload: nil, + MessageId: []byte("messageID"), + } + + msg.Sender = capabilityPeers[0][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + msg.Sender = capabilityPeers[1][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + response := <-request.ResponseChan() + require.Nil(t, response.Result) + require.NoError(t, response.Err) + }) + + t.Run("Unregister From Workflow Request with error", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + request, err := request.NewClientUnregisterFromWorkflowRequest(ctx, lggr, commoncap.UnregisterFromWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + }, + }, capInfo, workflowDonInfo, dispatcher, 10*time.Minute) + require.NoError(t, err) + defer request.Cancel(errors.New("test end")) + + <-dispatcher.msgs + <-dispatcher.msgs + assert.Equal(t, 0, len(dispatcher.msgs)) + + msg := &types.MessageBody{ + CapabilityId: capInfo.ID, + CapabilityDonId: capDonInfo.ID, + CallerDonId: workflowDonInfo.ID, + Method: types.MethodUnregisterFromWorkflow, + Payload: nil, + MessageId: []byte("messageID"), + Error: types.Error_INTERNAL_ERROR, + ErrorMsg: "an error", + } + + msg.Sender = capabilityPeers[0][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + msg.Sender = capabilityPeers[1][:] + err = request.OnMessage(ctx, msg) + require.NoError(t, err) + + response := <-request.ResponseChan() + require.Nil(t, response.Result) + assert.Equal(t, "an error", response.Err.Error()) + }) + } type clientRequestTestDispatcher struct { diff --git a/core/capabilities/remote/target/request/server_request.go b/core/capabilities/remote/executable/request/server_request.go similarity index 64% rename from core/capabilities/remote/target/request/server_request.go rename to core/capabilities/remote/executable/request/server_request.go index d23ba93a44d..f8bb2c33537 100644 --- a/core/capabilities/remote/target/request/server_request.go +++ b/core/capabilities/remote/executable/request/server_request.go @@ -23,7 +23,7 @@ type response struct { } type ServerRequest struct { - capability capabilities.TargetCapability + capability capabilities.ExecutableCapability capabilityPeerId p2ptypes.PeerID capabilityID string @@ -41,14 +41,16 @@ type ServerRequest struct { callingDon commoncap.DON requestMessageID string + method string requestTimeout time.Duration mux sync.Mutex lggr logger.Logger } -func NewServerRequest(capability capabilities.TargetCapability, capabilityID string, capabilityDonID uint32, capabilityPeerId p2ptypes.PeerID, - callingDon commoncap.DON, requestMessageID string, +func NewServerRequest(capability capabilities.ExecutableCapability, method string, capabilityID string, capabilityDonID uint32, + capabilityPeerId p2ptypes.PeerID, + callingDon commoncap.DON, requestID string, dispatcher types.Dispatcher, requestTimeout time.Duration, lggr logger.Logger) *ServerRequest { return &ServerRequest{ capability: capability, @@ -60,7 +62,8 @@ func NewServerRequest(capability capabilities.TargetCapability, capabilityID str requesters: map[p2ptypes.PeerID]bool{}, responseSentToRequester: map[p2ptypes.PeerID]bool{}, callingDon: callingDon, - requestMessageID: requestMessageID, + requestMessageID: requestID, + method: method, requestTimeout: requestTimeout, lggr: lggr.Named("ServerRequest"), } @@ -85,9 +88,18 @@ func (e *ServerRequest) OnMessage(ctx context.Context, msg *types.MessageBody) e e.lggr.Debugw("OnMessage called for request", "msgId", msg.MessageId, "calls", len(e.requesters), "hasResponse", e.response != nil) if e.minimumRequiredRequestsReceived() && !e.hasResponse() { - if err := e.executeRequest(ctx, msg.Payload); err != nil { - e.setError(types.Error_INTERNAL_ERROR, err.Error()) + + switch e.method { + case types.MethodExecute: + e.executeRequest(ctx, msg.Payload, executeCapabilityRequest) + case types.MethodRegisterToWorkflow: + e.executeRequest(ctx, msg.Payload, registerToWorkflow) + case types.MethodUnregisterFromWorkflow: + e.executeRequest(ctx, msg.Payload, unregisterFromWorkflow) + default: + e.setError(types.Error_INTERNAL_ERROR, fmt.Sprintf("unknown method %s", e.method)) } + } if err := e.sendResponses(); err != nil { @@ -115,31 +127,17 @@ func (e *ServerRequest) Cancel(err types.Error, msg string) error { return nil } -func (e *ServerRequest) executeRequest(ctx context.Context, payload []byte) error { +func (e *ServerRequest) executeRequest(ctx context.Context, payload []byte, method func(ctx context.Context, lggr logger.Logger, capability capabilities.ExecutableCapability, + payload []byte) ([]byte, error)) { ctxWithTimeout, cancel := context.WithTimeout(ctx, e.requestTimeout) defer cancel() - capabilityRequest, err := pb.UnmarshalCapabilityRequest(payload) - if err != nil { - return fmt.Errorf("failed to unmarshal capability request: %w", err) - } - - e.lggr.Debugw("executing capability", "metadata", capabilityRequest.Metadata) - capResponse, err := e.capability.Execute(ctxWithTimeout, capabilityRequest) - - if err != nil { - e.lggr.Debugw("received execution error", "workflowExecutionID", capabilityRequest.Metadata.WorkflowExecutionID, "error", err) - return fmt.Errorf("failed to execute capability: %w", err) - } - - responsePayload, err := pb.MarshalCapabilityResponse(capResponse) + responsePayload, err := method(ctxWithTimeout, e.lggr, e.capability, payload) if err != nil { - return fmt.Errorf("failed to marshal capability response: %w", err) + e.setError(types.Error_INTERNAL_ERROR, err.Error()) + } else { + e.setResult(responsePayload) } - - e.lggr.Debugw("received execution results", "workflowExecutionID", capabilityRequest.Metadata.WorkflowExecutionID) - e.setResult(responsePayload) - return nil } func (e *ServerRequest) addRequester(from p2ptypes.PeerID) error { @@ -227,3 +225,57 @@ func (e *ServerRequest) sendResponse(requester p2ptypes.PeerID) error { return nil } + +func executeCapabilityRequest(ctx context.Context, lggr logger.Logger, capability capabilities.ExecutableCapability, + payload []byte) ([]byte, error) { + capabilityRequest, err := pb.UnmarshalCapabilityRequest(payload) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal capability request: %w", err) + } + + lggr.Debugw("executing capability", "metadata", capabilityRequest.Metadata) + capResponse, err := capability.Execute(ctx, capabilityRequest) + + if err != nil { + lggr.Debugw("received execution error", "workflowExecutionID", capabilityRequest.Metadata.WorkflowExecutionID, "error", err) + return nil, fmt.Errorf("failed to execute capability: %w", err) + } + + responsePayload, err := pb.MarshalCapabilityResponse(capResponse) + if err != nil { + return nil, fmt.Errorf("failed to marshal capability response: %w", err) + } + + lggr.Debugw("received execution results", "workflowExecutionID", capabilityRequest.Metadata.WorkflowExecutionID) + return responsePayload, nil +} + +func registerToWorkflow(ctx context.Context, _ logger.Logger, capability capabilities.ExecutableCapability, + payload []byte) ([]byte, error) { + registerRequest, err := pb.UnmarshalRegisterToWorkflowRequest(payload) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal register to workflow request: %w", err) + } + + err = capability.RegisterToWorkflow(ctx, registerRequest) + if err != nil { + return nil, fmt.Errorf("failed to register to workflow: %w", err) + } + + return nil, nil +} + +func unregisterFromWorkflow(ctx context.Context, _ logger.Logger, capability capabilities.ExecutableCapability, + payload []byte) ([]byte, error) { + unregisterRequest, err := pb.UnmarshalUnregisterFromWorkflowRequest(payload) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal unregister from workflow request: %w", err) + } + + err = capability.UnregisterFromWorkflow(ctx, unregisterRequest) + if err != nil { + return nil, fmt.Errorf("failed to unregister from workflow: %w", err) + } + + return nil, nil +} diff --git a/core/capabilities/remote/target/request/server_request_test.go b/core/capabilities/remote/executable/request/server_request_test.go similarity index 61% rename from core/capabilities/remote/target/request/server_request_test.go rename to core/capabilities/remote/executable/request/server_request_test.go index b4bddbc955f..4ced59a8de7 100644 --- a/core/capabilities/remote/target/request/server_request_test.go +++ b/core/capabilities/remote/executable/request/server_request_test.go @@ -14,7 +14,7 @@ import ( commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" "github.com/smartcontractkit/chainlink-common/pkg/values" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target/request" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable/request" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" "github.com/smartcontractkit/chainlink/v2/core/logger" p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" @@ -58,7 +58,7 @@ func Test_ServerRequest_MessageValidation(t *testing.T) { require.NoError(t, err) t.Run("Send duplicate message", func(t *testing.T) { - req := request.NewServerRequest(capability, "capabilityID", 2, + req := request.NewServerRequest(capability, types.MethodExecute, "capabilityID", 2, capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) err := sendValidRequest(req, workflowPeers, capabilityPeerID, rawRequest) @@ -68,7 +68,7 @@ func Test_ServerRequest_MessageValidation(t *testing.T) { }) t.Run("Send message with non calling don peer", func(t *testing.T) { - req := request.NewServerRequest(capability, "capabilityID", 2, + req := request.NewServerRequest(capability, types.MethodExecute, "capabilityID", 2, capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) err := sendValidRequest(req, workflowPeers, capabilityPeerID, rawRequest) @@ -91,7 +91,7 @@ func Test_ServerRequest_MessageValidation(t *testing.T) { }) t.Run("Send message invalid payload", func(t *testing.T) { - req := request.NewServerRequest(capability, "capabilityID", 2, + req := request.NewServerRequest(capability, types.MethodExecute, "capabilityID", 2, capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) err := sendValidRequest(req, workflowPeers, capabilityPeerID, rawRequest) @@ -116,7 +116,7 @@ func Test_ServerRequest_MessageValidation(t *testing.T) { t.Run("Send second valid request when capability errors", func(t *testing.T) { dispatcher := &testDispatcher{} - req := request.NewServerRequest(TestErrorCapability{}, "capabilityID", 2, + req := request.NewServerRequest(TestErrorCapability{}, types.MethodExecute, "capabilityID", 2, capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) err := sendValidRequest(req, workflowPeers, capabilityPeerID, rawRequest) @@ -141,9 +141,9 @@ func Test_ServerRequest_MessageValidation(t *testing.T) { assert.Equal(t, dispatcher.msgs[1].ErrorMsg, "failed to execute capability: an error") }) - t.Run("Send second valid request", func(t *testing.T) { + t.Run("Execute capability", func(t *testing.T) { dispatcher := &testDispatcher{} - request := request.NewServerRequest(capability, "capabilityID", 2, + request := request.NewServerRequest(capability, types.MethodExecute, "capabilityID", 2, capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) err := sendValidRequest(request, workflowPeers, capabilityPeerID, rawRequest) @@ -165,6 +165,108 @@ func Test_ServerRequest_MessageValidation(t *testing.T) { assert.Equal(t, dispatcher.msgs[0].Error, types.Error_OK) assert.Equal(t, dispatcher.msgs[1].Error, types.Error_OK) }) + t.Run("Register to workflow request", func(t *testing.T) { + dispatcher := &testDispatcher{} + request := request.NewServerRequest(capability, types.MethodRegisterToWorkflow, "capabilityID", 2, + capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) + + err := sendValidRequest(request, workflowPeers, capabilityPeerID, rawRequest) + require.NoError(t, err) + + err = request.OnMessage(context.Background(), &types.MessageBody{ + Version: 0, + Sender: workflowPeers[1][:], + Receiver: capabilityPeerID[:], + MessageId: []byte("workflowID" + "workflowExecutionID"), + CapabilityId: "capabilityID", + CapabilityDonId: 2, + CallerDonId: 1, + Method: types.MethodRegisterToWorkflow, + Payload: rawRequest, + }) + assert.NoError(t, err) + assert.Equal(t, 2, len(dispatcher.msgs)) + assert.Equal(t, dispatcher.msgs[0].Error, types.Error_OK) + assert.Equal(t, dispatcher.msgs[1].Error, types.Error_OK) + }) + t.Run("Register to workflow request errors", func(t *testing.T) { + dispatcher := &testDispatcher{} + req := request.NewServerRequest(TestErrorCapability{}, types.MethodRegisterToWorkflow, "capabilityID", 2, + capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) + + err := sendValidRequest(req, workflowPeers, capabilityPeerID, rawRequest) + require.NoError(t, err) + + err = req.OnMessage(context.Background(), &types.MessageBody{ + Version: 0, + Sender: workflowPeers[1][:], + Receiver: capabilityPeerID[:], + MessageId: []byte("workflowID" + "workflowExecutionID"), + CapabilityId: "capabilityID", + CapabilityDonId: 2, + CallerDonId: 1, + Method: types.MethodRegisterToWorkflow, + Payload: rawRequest, + }) + assert.NoError(t, err) + assert.Equal(t, 2, len(dispatcher.msgs)) + assert.Equal(t, dispatcher.msgs[0].Error, types.Error_INTERNAL_ERROR) + assert.Equal(t, dispatcher.msgs[0].ErrorMsg, "failed to register to workflow: an error") + assert.Equal(t, dispatcher.msgs[1].Error, types.Error_INTERNAL_ERROR) + assert.Equal(t, dispatcher.msgs[1].ErrorMsg, "failed to register to workflow: an error") + }) + t.Run("Unregister from workflow request", func(t *testing.T) { + dispatcher := &testDispatcher{} + request := request.NewServerRequest(capability, types.MethodUnregisterFromWorkflow, "capabilityID", 2, + capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) + + err := sendValidRequest(request, workflowPeers, capabilityPeerID, rawRequest) + require.NoError(t, err) + + err = request.OnMessage(context.Background(), &types.MessageBody{ + Version: 0, + Sender: workflowPeers[1][:], + Receiver: capabilityPeerID[:], + MessageId: []byte("workflowID" + "workflowExecutionID"), + CapabilityId: "capabilityID", + CapabilityDonId: 2, + CallerDonId: 1, + Method: types.MethodUnregisterFromWorkflow, + Payload: rawRequest, + }) + assert.NoError(t, err) + assert.Equal(t, 2, len(dispatcher.msgs)) + assert.Equal(t, dispatcher.msgs[0].Error, types.Error_OK) + assert.Equal(t, dispatcher.msgs[1].Error, types.Error_OK) + }) + + t.Run("Unregister from workflow request errors", func(t *testing.T) { + dispatcher := &testDispatcher{} + req := request.NewServerRequest(TestErrorCapability{}, types.MethodUnregisterFromWorkflow, "capabilityID", 2, + capabilityPeerID, callingDon, "requestMessageID", dispatcher, 10*time.Minute, lggr) + + err := sendValidRequest(req, workflowPeers, capabilityPeerID, rawRequest) + require.NoError(t, err) + + err = req.OnMessage(context.Background(), &types.MessageBody{ + Version: 0, + Sender: workflowPeers[1][:], + Receiver: capabilityPeerID[:], + MessageId: []byte("workflowID" + "workflowExecutionID"), + CapabilityId: "capabilityID", + CapabilityDonId: 2, + CallerDonId: 1, + Method: types.MethodUnregisterFromWorkflow, + Payload: rawRequest, + }) + assert.NoError(t, err) + assert.Equal(t, 2, len(dispatcher.msgs)) + assert.Equal(t, dispatcher.msgs[0].Error, types.Error_INTERNAL_ERROR) + assert.Equal(t, dispatcher.msgs[0].ErrorMsg, "failed to unregister from workflow: an error") + assert.Equal(t, dispatcher.msgs[1].Error, types.Error_INTERNAL_ERROR) + assert.Equal(t, dispatcher.msgs[1].ErrorMsg, "failed to unregister from workflow: an error") + }) + } type serverRequest interface { @@ -261,6 +363,14 @@ func (t TestErrorCapability) Execute(ctx context.Context, request commoncap.Capa return commoncap.CapabilityResponse{}, errors.New("an error") } +func (t TestErrorCapability) RegisterToWorkflow(ctx context.Context, request commoncap.RegisterToWorkflowRequest) error { + return errors.New("an error") +} + +func (t TestErrorCapability) UnregisterFromWorkflow(ctx context.Context, request commoncap.UnregisterFromWorkflowRequest) error { + return errors.New("an error") +} + func NewP2PPeerID(t *testing.T) p2ptypes.PeerID { id := p2ptypes.PeerID{} require.NoError(t, id.UnmarshalText([]byte(NewPeerID()))) diff --git a/core/capabilities/remote/target/server.go b/core/capabilities/remote/executable/server.go similarity index 81% rename from core/capabilities/remote/target/server.go rename to core/capabilities/remote/executable/server.go index 5324475b192..267833fb1ff 100644 --- a/core/capabilities/remote/target/server.go +++ b/core/capabilities/remote/executable/server.go @@ -1,4 +1,4 @@ -package target +package executable import ( "context" @@ -12,7 +12,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target/request" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable/request" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" "github.com/smartcontractkit/chainlink/v2/core/capabilities/validation" p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" @@ -20,9 +20,9 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" ) -// server manages all external users of a local target capability. +// server manages all external users of a local executable capability. // Its responsibilities are: -// 1. Manage requests from external nodes executing the target capability once sufficient requests are received. +// 1. Manage requests from external nodes executing the executable capability once sufficient requests are received. // 2. Send out responses produced by an underlying capability to all requesters. // // server communicates with corresponding client on remote nodes. @@ -30,9 +30,9 @@ type server struct { services.StateMachine lggr logger.Logger - config *commoncap.RemoteTargetConfig + config *commoncap.RemoteExecutableConfig peerID p2ptypes.PeerID - underlying commoncap.TargetCapability + underlying commoncap.ExecutableCapability capInfo commoncap.CapabilityInfo localDonInfo commoncap.DON workflowDONs map[uint32]commoncap.DON @@ -57,14 +57,15 @@ type requestAndMsgID struct { messageID string } -func NewServer(config *commoncap.RemoteTargetConfig, peerID p2ptypes.PeerID, underlying commoncap.TargetCapability, capInfo commoncap.CapabilityInfo, localDonInfo commoncap.DON, +func NewServer(remoteExecutableConfig *commoncap.RemoteExecutableConfig, peerID p2ptypes.PeerID, underlying commoncap.ExecutableCapability, + capInfo commoncap.CapabilityInfo, localDonInfo commoncap.DON, workflowDONs map[uint32]commoncap.DON, dispatcher types.Dispatcher, requestTimeout time.Duration, lggr logger.Logger) *server { - if config == nil { - lggr.Info("no config provided, using default values") - config = &commoncap.RemoteTargetConfig{} + if remoteExecutableConfig == nil { + lggr.Info("no remote config provided, using default values") + remoteExecutableConfig = &commoncap.RemoteExecutableConfig{} } return &server{ - config: config, + config: remoteExecutableConfig, underlying: underlying, peerID: peerID, capInfo: capInfo, @@ -76,7 +77,7 @@ func NewServer(config *commoncap.RemoteTargetConfig, peerID p2ptypes.PeerID, und messageIDToRequestIDsCount: map[string]map[string]int{}, requestTimeout: requestTimeout, - lggr: lggr.Named("TargetServer"), + lggr: lggr.Named("ExecutableCapabilityServer"), stopCh: make(services.StopChan), } } @@ -88,7 +89,7 @@ func (r *server) Start(ctx context.Context) error { defer r.wg.Done() ticker := time.NewTicker(r.requestTimeout) defer ticker.Stop() - r.lggr.Info("TargetServer started") + r.lggr.Info("executable capability server started") for { select { case <-r.stopCh: @@ -106,7 +107,7 @@ func (r *server) Close() error { return r.StopOnce(r.Name(), func() error { close(r.stopCh) r.wg.Wait() - r.lggr.Info("TargetServer closed") + r.lggr.Info("executable capability server closed") return nil }) } @@ -131,9 +132,10 @@ func (r *server) Receive(ctx context.Context, msg *types.MessageBody) { r.receiveLock.Lock() defer r.receiveLock.Unlock() - if msg.Method != types.MethodExecute { + switch msg.Method { + case types.MethodExecute, types.MethodRegisterToWorkflow, types.MethodUnregisterFromWorkflow: + default: r.lggr.Errorw("received request for unsupported method type", "method", remote.SanitizeLogString(msg.Method)) - return } messageId, err := GetMessageID(msg) @@ -175,7 +177,7 @@ func (r *server) Receive(ctx context.Context, msg *types.MessageBody) { } r.requestIDToRequest[requestID] = requestAndMsgID{ - request: request.NewServerRequest(r.underlying, r.capInfo.ID, r.localDonInfo.ID, r.peerID, + request: request.NewServerRequest(r.underlying, msg.Method, r.capInfo.ID, r.localDonInfo.ID, r.peerID, callingDon, messageId, r.dispatcher, r.requestTimeout, r.lggr), messageID: messageId, } @@ -196,8 +198,10 @@ func (r *server) getMessageHash(msg *types.MessageBody) ([32]byte, error) { } for _, path := range r.config.RequestHashExcludedAttributes { - if !req.Inputs.DeleteAtPath(path) { - return [32]byte{}, fmt.Errorf("failed to delete attribute from map at path: %s", path) + if req.Inputs != nil { + if !req.Inputs.DeleteAtPath(path) { + return [32]byte{}, fmt.Errorf("failed to delete attribute from map at path: %s", path) + } } } @@ -226,5 +230,5 @@ func (r *server) HealthReport() map[string]error { } func (r *server) Name() string { - return "TargetServer" + return "ExecutableServer" } diff --git a/core/capabilities/remote/target/server_test.go b/core/capabilities/remote/executable/server_test.go similarity index 59% rename from core/capabilities/remote/target/server_test.go rename to core/capabilities/remote/executable/server_test.go index 505a2dcce5d..73163c711e8 100644 --- a/core/capabilities/remote/target/server_test.go +++ b/core/capabilities/remote/executable/server_test.go @@ -1,4 +1,4 @@ -package target_test +package executable_test import ( "context" @@ -13,7 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/values" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable" remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -25,7 +25,7 @@ func Test_Server_ExcludesNonDeterministicInputAttributes(t *testing.T) { numCapabilityPeers := 4 - callers, srvcs := testRemoteTargetServer(ctx, t, &commoncap.RemoteTargetConfig{RequestHashExcludedAttributes: []string{"signed_report.Signatures"}}, + callers, srvcs := testRemoteExecutableCapabilityServer(ctx, t, &commoncap.RemoteExecutableConfig{RequestHashExcludedAttributes: []string{"signed_report.Signatures"}}, &TestCapability{}, 10, 9, numCapabilityPeers, 3, 10*time.Minute) for idx, caller := range callers { @@ -56,12 +56,12 @@ func Test_Server_ExcludesNonDeterministicInputAttributes(t *testing.T) { closeServices(t, srvcs) } -func Test_Server_RespondsAfterSufficientRequests(t *testing.T) { +func Test_Server_Execute_RespondsAfterSufficientRequests(t *testing.T) { ctx := testutils.Context(t) numCapabilityPeers := 4 - callers, srvcs := testRemoteTargetServer(ctx, t, &commoncap.RemoteTargetConfig{}, &TestCapability{}, 10, 9, numCapabilityPeers, 3, 10*time.Minute) + callers, srvcs := testRemoteExecutableCapabilityServer(ctx, t, &commoncap.RemoteExecutableConfig{}, &TestCapability{}, 10, 9, numCapabilityPeers, 3, 10*time.Minute) for _, caller := range callers { _, err := caller.Execute(context.Background(), @@ -83,12 +83,94 @@ func Test_Server_RespondsAfterSufficientRequests(t *testing.T) { closeServices(t, srvcs) } +func Test_Server_RegisterToWorkflow_RespondsAfterSufficientRequests(t *testing.T) { + ctx := testutils.Context(t) + + numCapabilityPeers := 4 + + callers, srvcs := testRemoteExecutableCapabilityServer(ctx, t, &commoncap.RemoteExecutableConfig{}, &TestCapability{}, 10, 9, numCapabilityPeers, 3, 10*time.Minute) + + for _, caller := range callers { + + err := caller.RegisterToWorkflow(context.Background(), commoncap.RegisterToWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + ReferenceID: stepReferenceID1, + }, + }) + + require.NoError(t, err) + } + + for _, caller := range callers { + for i := 0; i < numCapabilityPeers; i++ { + msg := <-caller.receivedMessages + assert.Equal(t, remotetypes.Error_OK, msg.Error) + } + } + closeServices(t, srvcs) +} + +func Test_Server_RegisterToWorkflow_Error(t *testing.T) { + ctx := testutils.Context(t) + + numCapabilityPeers := 4 + + callers, srvcs := testRemoteExecutableCapabilityServer(ctx, t, &commoncap.RemoteExecutableConfig{}, &TestErrorCapability{}, 10, 9, numCapabilityPeers, 3, 10*time.Minute) + + for _, caller := range callers { + + err := caller.RegisterToWorkflow(context.Background(), commoncap.RegisterToWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + ReferenceID: stepReferenceID1, + }, + }) + + require.NoError(t, err) + } + + for _, caller := range callers { + for i := 0; i < numCapabilityPeers; i++ { + msg := <-caller.receivedMessages + assert.Equal(t, remotetypes.Error_INTERNAL_ERROR, msg.Error) + } + } + closeServices(t, srvcs) +} + +func Test_Server_UnregisterFromWorkflow_RespondsAfterSufficientRequests(t *testing.T) { + ctx := testutils.Context(t) + + numCapabilityPeers := 4 + + callers, srvcs := testRemoteExecutableCapabilityServer(ctx, t, &commoncap.RemoteExecutableConfig{}, &TestCapability{}, 10, 9, numCapabilityPeers, 3, 10*time.Minute) + + for _, caller := range callers { + err := caller.UnregisterFromWorkflow(context.Background(), commoncap.UnregisterFromWorkflowRequest{ + Metadata: commoncap.RegistrationMetadata{ + WorkflowID: workflowID1, + ReferenceID: stepReferenceID1, + }, + }) + require.NoError(t, err) + } + + for _, caller := range callers { + for i := 0; i < numCapabilityPeers; i++ { + msg := <-caller.receivedMessages + assert.Equal(t, remotetypes.Error_OK, msg.Error) + } + } + closeServices(t, srvcs) +} + func Test_Server_InsufficientCallers(t *testing.T) { ctx := testutils.Context(t) numCapabilityPeers := 4 - callers, srvcs := testRemoteTargetServer(ctx, t, &commoncap.RemoteTargetConfig{}, &TestCapability{}, 10, 10, numCapabilityPeers, 3, 100*time.Millisecond) + callers, srvcs := testRemoteExecutableCapabilityServer(ctx, t, &commoncap.RemoteExecutableConfig{}, &TestCapability{}, 10, 10, numCapabilityPeers, 3, 100*time.Millisecond) for _, caller := range callers { _, err := caller.Execute(context.Background(), @@ -115,7 +197,7 @@ func Test_Server_CapabilityError(t *testing.T) { numCapabilityPeers := 4 - callers, srvcs := testRemoteTargetServer(ctx, t, &commoncap.RemoteTargetConfig{}, &TestErrorCapability{}, 10, 9, numCapabilityPeers, 3, 100*time.Millisecond) + callers, srvcs := testRemoteExecutableCapabilityServer(ctx, t, &commoncap.RemoteExecutableConfig{}, &TestErrorCapability{}, 10, 9, numCapabilityPeers, 3, 100*time.Millisecond) for _, caller := range callers { _, err := caller.Execute(context.Background(), @@ -137,9 +219,9 @@ func Test_Server_CapabilityError(t *testing.T) { closeServices(t, srvcs) } -func testRemoteTargetServer(ctx context.Context, t *testing.T, - config *commoncap.RemoteTargetConfig, - underlying commoncap.TargetCapability, +func testRemoteExecutableCapabilityServer(ctx context.Context, t *testing.T, + config *commoncap.RemoteExecutableConfig, + underlying commoncap.ExecutableCapability, numWorkflowPeers int, workflowDonF uint8, numCapabilityPeers int, capabilityDonF uint8, capabilityNodeResponseTimeout time.Duration) ([]*serverTestClient, []services.Service) { lggr := logger.TestLogger(t) @@ -189,7 +271,7 @@ func testRemoteTargetServer(ctx context.Context, t *testing.T, for i := 0; i < numCapabilityPeers; i++ { capabilityPeer := capabilityPeers[i] capabilityDispatcher := broker.NewDispatcherForNode(capabilityPeer) - capabilityNode := target.NewServer(config, capabilityPeer, underlying, capInfo, capDonInfo, workflowDONs, capabilityDispatcher, + capabilityNode := executable.NewServer(config, capabilityPeer, underlying, capInfo, capDonInfo, workflowDONs, capabilityDispatcher, capabilityNodeResponseTimeout, lggr) require.NoError(t, capabilityNode.Start(ctx)) broker.RegisterReceiverNode(capabilityPeer, capabilityNode) @@ -236,12 +318,60 @@ func (r *serverTestClient) Info(ctx context.Context) (commoncap.CapabilityInfo, panic("not implemented") } -func (r *serverTestClient) RegisterToWorkflow(ctx context.Context, request commoncap.RegisterToWorkflowRequest) error { - panic("not implemented") +func (r *serverTestClient) RegisterToWorkflow(ctx context.Context, req commoncap.RegisterToWorkflowRequest) error { + rawRequest, err := pb.MarshalRegisterToWorkflowRequest(req) + if err != nil { + return err + } + + messageID := remotetypes.MethodRegisterToWorkflow + ":" + req.Metadata.WorkflowID + + for _, node := range r.capabilityDonInfo.Members { + message := &remotetypes.MessageBody{ + CapabilityId: "capability-id", + CapabilityDonId: 1, + CallerDonId: 2, + Method: remotetypes.MethodRegisterToWorkflow, + Payload: rawRequest, + MessageId: []byte(messageID), + Sender: r.peerID[:], + Receiver: node[:], + } + + if err = r.dispatcher.Send(node, message); err != nil { + return err + } + } + + return nil } -func (r *serverTestClient) UnregisterFromWorkflow(ctx context.Context, request commoncap.UnregisterFromWorkflowRequest) error { - panic("not implemented") +func (r *serverTestClient) UnregisterFromWorkflow(ctx context.Context, req commoncap.UnregisterFromWorkflowRequest) error { + rawRequest, err := pb.MarshalUnregisterFromWorkflowRequest(req) + if err != nil { + return err + } + + messageID := remotetypes.MethodUnregisterFromWorkflow + ":" + req.Metadata.WorkflowID + + for _, node := range r.capabilityDonInfo.Members { + message := &remotetypes.MessageBody{ + CapabilityId: "capability-id", + CapabilityDonId: 1, + CallerDonId: 2, + Method: remotetypes.MethodUnregisterFromWorkflow, + Payload: rawRequest, + MessageId: []byte(messageID), + Sender: r.peerID[:], + Receiver: node[:], + } + + if err = r.dispatcher.Send(node, message); err != nil { + return err + } + } + + return nil } func (r *serverTestClient) Execute(ctx context.Context, req commoncap.CapabilityRequest) (<-chan commoncap.CapabilityResponse, error) { @@ -250,10 +380,7 @@ func (r *serverTestClient) Execute(ctx context.Context, req commoncap.Capability return nil, err } - messageID, err := target.GetMessageIDForRequest(req) - if err != nil { - return nil, err - } + messageID := remotetypes.MethodExecute + ":" + req.Metadata.WorkflowExecutionID for _, node := range r.capabilityDonInfo.Members { message := &remotetypes.MessageBody{ diff --git a/core/capabilities/remote/target/client_test.go b/core/capabilities/remote/target/client_test.go deleted file mode 100644 index 697cb6e383a..00000000000 --- a/core/capabilities/remote/target/client_test.go +++ /dev/null @@ -1,316 +0,0 @@ -package target_test - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" - "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" - "github.com/smartcontractkit/chainlink-common/pkg/values" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/target" - remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/transmission" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" -) - -const ( - workflowID1 = "15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0" - workflowExecutionID1 = "95ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0abbadeed" -) - -func Test_Client_DonTopologies(t *testing.T) { - ctx := testutils.Context(t) - - transmissionSchedule, err := values.NewMap(map[string]any{ - "schedule": transmission.Schedule_OneAtATime, - "deltaStage": "10ms", - }) - require.NoError(t, err) - - responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { - require.NoError(t, responseError) - mp, err := response.Value.Unwrap() - require.NoError(t, err) - assert.Equal(t, "aValue1", mp.(map[string]any)["response"].(string)) - } - - capability := &TestCapability{} - - responseTimeOut := 10 * time.Minute - - testClient(ctx, t, 1, responseTimeOut, 1, 0, - capability, transmissionSchedule, responseTest) - - testClient(ctx, t, 10, responseTimeOut, 1, 0, - capability, transmissionSchedule, responseTest) - - testClient(ctx, t, 1, responseTimeOut, 10, 3, - capability, transmissionSchedule, responseTest) - - testClient(ctx, t, 10, responseTimeOut, 10, 3, - capability, transmissionSchedule, responseTest) - - testClient(ctx, t, 10, responseTimeOut, 10, 9, - capability, transmissionSchedule, responseTest) -} - -func Test_Client_TransmissionSchedules(t *testing.T) { - ctx := testutils.Context(t) - - responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { - require.NoError(t, responseError) - mp, err := response.Value.Unwrap() - require.NoError(t, err) - assert.Equal(t, "aValue1", mp.(map[string]any)["response"].(string)) - } - - capability := &TestCapability{} - - responseTimeOut := 10 * time.Minute - - transmissionSchedule, err := values.NewMap(map[string]any{ - "schedule": transmission.Schedule_OneAtATime, - "deltaStage": "10ms", - }) - require.NoError(t, err) - - testClient(ctx, t, 1, responseTimeOut, 1, 0, - capability, transmissionSchedule, responseTest) - testClient(ctx, t, 10, responseTimeOut, 10, 3, - capability, transmissionSchedule, responseTest) - - transmissionSchedule, err = values.NewMap(map[string]any{ - "schedule": transmission.Schedule_AllAtOnce, - "deltaStage": "10ms", - }) - require.NoError(t, err) - - testClient(ctx, t, 1, responseTimeOut, 1, 0, - capability, transmissionSchedule, responseTest) - testClient(ctx, t, 10, responseTimeOut, 10, 3, - capability, transmissionSchedule, responseTest) -} - -func Test_Client_TimesOutIfInsufficientCapabilityPeerResponses(t *testing.T) { - ctx := testutils.Context(t) - - responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) { - assert.NotNil(t, responseError) - } - - capability := &TestCapability{} - - transmissionSchedule, err := values.NewMap(map[string]any{ - "schedule": transmission.Schedule_AllAtOnce, - "deltaStage": "10ms", - }) - require.NoError(t, err) - - // number of capability peers is less than F + 1 - - testClient(ctx, t, 10, 1*time.Second, 10, 11, - capability, transmissionSchedule, responseTest) -} - -func testClient(ctx context.Context, t *testing.T, numWorkflowPeers int, workflowNodeResponseTimeout time.Duration, - numCapabilityPeers int, capabilityDonF uint8, underlying commoncap.TargetCapability, transmissionSchedule *values.Map, - responseTest func(t *testing.T, responseCh commoncap.CapabilityResponse, responseError error)) { - lggr := logger.TestLogger(t) - - capabilityPeers := make([]p2ptypes.PeerID, numCapabilityPeers) - for i := 0; i < numCapabilityPeers; i++ { - capabilityPeers[i] = NewP2PPeerID(t) - } - - capDonInfo := commoncap.DON{ - ID: 1, - Members: capabilityPeers, - F: capabilityDonF, - } - - capInfo := commoncap.CapabilityInfo{ - ID: "cap_id@1.0.0", - CapabilityType: commoncap.CapabilityTypeTarget, - Description: "Remote Target", - DON: &capDonInfo, - } - - workflowPeers := make([]p2ptypes.PeerID, numWorkflowPeers) - for i := 0; i < numWorkflowPeers; i++ { - workflowPeers[i] = NewP2PPeerID(t) - } - - workflowDonInfo := commoncap.DON{ - Members: workflowPeers, - ID: 2, - } - - broker := newTestAsyncMessageBroker(t, 100) - - receivers := make([]remotetypes.Receiver, numCapabilityPeers) - for i := 0; i < numCapabilityPeers; i++ { - capabilityDispatcher := broker.NewDispatcherForNode(capabilityPeers[i]) - receiver := newTestServer(capabilityPeers[i], capabilityDispatcher, workflowDonInfo, underlying) - broker.RegisterReceiverNode(capabilityPeers[i], receiver) - receivers[i] = receiver - } - - callers := make([]commoncap.TargetCapability, numWorkflowPeers) - - for i := 0; i < numWorkflowPeers; i++ { - workflowPeerDispatcher := broker.NewDispatcherForNode(workflowPeers[i]) - caller := target.NewClient(capInfo, workflowDonInfo, workflowPeerDispatcher, workflowNodeResponseTimeout, lggr) - servicetest.Run(t, caller) - broker.RegisterReceiverNode(workflowPeers[i], caller) - callers[i] = caller - } - - servicetest.Run(t, broker) - - executeInputs, err := values.NewMap( - map[string]any{ - "executeValue1": "aValue1", - }, - ) - - require.NoError(t, err) - - wg := &sync.WaitGroup{} - wg.Add(len(callers)) - - // Fire off all the requests - for _, caller := range callers { - go func(caller commoncap.TargetCapability) { - defer wg.Done() - responseCh, err := caller.Execute(ctx, - commoncap.CapabilityRequest{ - Metadata: commoncap.RequestMetadata{ - WorkflowID: workflowID1, - WorkflowExecutionID: workflowExecutionID1, - }, - Config: transmissionSchedule, - Inputs: executeInputs, - }) - - responseTest(t, responseCh, err) - }(caller) - } - - wg.Wait() -} - -// Simple client that only responds once it has received a message from each workflow peer -type clientTestServer struct { - peerID p2ptypes.PeerID - dispatcher remotetypes.Dispatcher - workflowDonInfo commoncap.DON - messageIDToSenders map[string]map[p2ptypes.PeerID]bool - - targetCapability commoncap.TargetCapability - - mux sync.Mutex -} - -func newTestServer(peerID p2ptypes.PeerID, dispatcher remotetypes.Dispatcher, workflowDonInfo commoncap.DON, - targetCapability commoncap.TargetCapability) *clientTestServer { - return &clientTestServer{ - dispatcher: dispatcher, - workflowDonInfo: workflowDonInfo, - peerID: peerID, - messageIDToSenders: make(map[string]map[p2ptypes.PeerID]bool), - targetCapability: targetCapability, - } -} - -func (t *clientTestServer) Receive(_ context.Context, msg *remotetypes.MessageBody) { - t.mux.Lock() - defer t.mux.Unlock() - - sender := toPeerID(msg.Sender) - messageID, err := target.GetMessageID(msg) - if err != nil { - panic(err) - } - - if t.messageIDToSenders[messageID] == nil { - t.messageIDToSenders[messageID] = make(map[p2ptypes.PeerID]bool) - } - - sendersOfMessageID := t.messageIDToSenders[messageID] - if sendersOfMessageID[sender] { - panic("received duplicate message") - } - - sendersOfMessageID[sender] = true - - if len(t.messageIDToSenders[messageID]) == len(t.workflowDonInfo.Members) { - capabilityRequest, err := pb.UnmarshalCapabilityRequest(msg.Payload) - if err != nil { - panic(err) - } - - resp, responseErr := t.targetCapability.Execute(context.Background(), capabilityRequest) - - for receiver := range t.messageIDToSenders[messageID] { - var responseMsg = &remotetypes.MessageBody{ - CapabilityId: "cap_id@1.0.0", - CapabilityDonId: 1, - CallerDonId: t.workflowDonInfo.ID, - Method: remotetypes.MethodExecute, - MessageId: []byte(messageID), - Sender: t.peerID[:], - Receiver: receiver[:], - } - - if responseErr != nil { - responseMsg.Error = remotetypes.Error_INTERNAL_ERROR - } else { - payload, marshalErr := pb.MarshalCapabilityResponse(resp) - if marshalErr != nil { - panic(marshalErr) - } - responseMsg.Payload = payload - } - - err = t.dispatcher.Send(receiver, responseMsg) - if err != nil { - panic(err) - } - } - } -} - -type TestDispatcher struct { - sentMessagesCh chan *remotetypes.MessageBody - receiver remotetypes.Receiver -} - -func NewTestDispatcher() *TestDispatcher { - return &TestDispatcher{ - sentMessagesCh: make(chan *remotetypes.MessageBody, 1), - } -} - -func (t *TestDispatcher) SendToReceiver(msgBody *remotetypes.MessageBody) { - t.receiver.Receive(context.Background(), msgBody) -} - -func (t *TestDispatcher) SetReceiver(capabilityId string, donId string, receiver remotetypes.Receiver) error { - t.receiver = receiver - return nil -} - -func (t *TestDispatcher) RemoveReceiver(capabilityId string, donId string) {} - -func (t *TestDispatcher) Send(peerID p2ptypes.PeerID, msgBody *remotetypes.MessageBody) error { - t.sentMessagesCh <- msgBody - return nil -} diff --git a/core/capabilities/remote/types/types.go b/core/capabilities/remote/types/types.go index 54ec16f09f1..fefc9a9b5fe 100644 --- a/core/capabilities/remote/types/types.go +++ b/core/capabilities/remote/types/types.go @@ -13,10 +13,12 @@ import ( ) const ( - MethodRegisterTrigger = "RegisterTrigger" - MethodUnRegisterTrigger = "UnregisterTrigger" - MethodTriggerEvent = "TriggerEvent" - MethodExecute = "Execute" + MethodRegisterTrigger = "RegisterTrigger" + MethodUnRegisterTrigger = "UnregisterTrigger" + MethodTriggerEvent = "TriggerEvent" + MethodExecute = "Execute" + MethodRegisterToWorkflow = "RegisterToWorkflow" + MethodUnregisterFromWorkflow = "UnregisterFromWorkflow" ) type Dispatcher interface { diff --git a/core/capabilities/transmission/local_target_capability_test.go b/core/capabilities/transmission/local_target_capability_test.go index 67f22753bda..e1057ed3f8d 100644 --- a/core/capabilities/transmission/local_target_capability_test.go +++ b/core/capabilities/transmission/local_target_capability_test.go @@ -61,15 +61,15 @@ func TestScheduledExecutionStrategy_LocalDON(t *testing.T) { name: "position 1; oneAtATime", position: 1, schedule: "oneAtATime", - low: 100 * time.Millisecond, - high: 300 * time.Millisecond, + low: 300 * time.Millisecond, + high: 400 * time.Millisecond, }, { name: "position 2; oneAtATime", position: 2, schedule: "oneAtATime", - low: 300 * time.Millisecond, - high: 400 * time.Millisecond, + low: 100 * time.Millisecond, + high: 200 * time.Millisecond, }, { name: "position 3; oneAtATime", diff --git a/core/capabilities/transmission/transmission.go b/core/capabilities/transmission/transmission.go index 88ce0fa3edd..5015792f24e 100644 --- a/core/capabilities/transmission/transmission.go +++ b/core/capabilities/transmission/transmission.go @@ -4,9 +4,8 @@ import ( "fmt" "time" - "github.com/smartcontractkit/libocr/permutation" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/validation" + "github.com/smartcontractkit/libocr/permutation" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/values" @@ -27,7 +26,7 @@ type TransmissionConfig struct { DeltaStage time.Duration } -func extractTransmissionConfig(config *values.Map) (TransmissionConfig, error) { +func ExtractTransmissionConfig(config *values.Map) (TransmissionConfig, error) { var tc struct { DeltaStage string Schedule string @@ -37,6 +36,14 @@ func extractTransmissionConfig(config *values.Map) (TransmissionConfig, error) { return TransmissionConfig{}, fmt.Errorf("failed to unwrap tranmission config from value map: %w", err) } + // Default if no schedule and deltaStage is provided + if len(tc.Schedule) == 0 && len(tc.DeltaStage) == 0 { + return TransmissionConfig{ + Schedule: Schedule_AllAtOnce, + DeltaStage: 0, + }, nil + } + duration, err := time.ParseDuration(tc.DeltaStage) if err != nil { return TransmissionConfig{}, fmt.Errorf("failed to parse DeltaStage %s as duration: %w", tc.DeltaStage, err) @@ -51,20 +58,20 @@ func extractTransmissionConfig(config *values.Map) (TransmissionConfig, error) { // GetPeerIDToTransmissionDelay returns a map of PeerID to the time.Duration that the node with that PeerID should wait // before transmitting the capability request. If a node is not in the map, it should not transmit. func GetPeerIDToTransmissionDelay(donPeerIDs []types.PeerID, req capabilities.CapabilityRequest) (map[types.PeerID]time.Duration, error) { - tc, err := extractTransmissionConfig(req.Config) + tc, err := ExtractTransmissionConfig(req.Config) if err != nil { return nil, fmt.Errorf("failed to extract transmission config from request: %w", err) } - if err = validation.ValidateWorkflowOrExecutionID(req.Metadata.WorkflowID); err != nil { - return nil, fmt.Errorf("workflow ID is invalid: %w", err) + workflowExecutionID := req.Metadata.WorkflowExecutionID + if err := validation.ValidateWorkflowOrExecutionID(workflowExecutionID); err != nil { + return nil, fmt.Errorf("workflow or execution ID is invalid: %w", err) } - if err = validation.ValidateWorkflowOrExecutionID(req.Metadata.WorkflowExecutionID); err != nil { - return nil, fmt.Errorf("workflow execution ID is invalid: %w", err) - } + return GetPeerIDToTransmissionDelaysForConfig(donPeerIDs, workflowExecutionID, tc) +} - transmissionID := req.Metadata.WorkflowID + req.Metadata.WorkflowExecutionID +func GetPeerIDToTransmissionDelaysForConfig(donPeerIDs []types.PeerID, transmissionID string, tc TransmissionConfig) (map[types.PeerID]time.Duration, error) { donMemberCount := len(donPeerIDs) key := transmissionScheduleSeed(transmissionID) diff --git a/core/capabilities/transmission/transmission_test.go b/core/capabilities/transmission/transmission_test.go index aaa367e78cf..1cb91c364a5 100644 --- a/core/capabilities/transmission/transmission_test.go +++ b/core/capabilities/transmission/transmission_test.go @@ -38,10 +38,10 @@ func Test_GetPeerIDToTransmissionDelay(t *testing.T) { "100ms", "15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0", map[string]time.Duration{ - "one": 300 * time.Millisecond, - "two": 0 * time.Millisecond, - "three": 100 * time.Millisecond, - "four": 200 * time.Millisecond, + "one": 100 * time.Millisecond, + "two": 200 * time.Millisecond, + "three": 300 * time.Millisecond, + "four": 0 * time.Millisecond, }, }, @@ -66,10 +66,10 @@ func Test_GetPeerIDToTransmissionDelay(t *testing.T) { "100ms", "16c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce1", map[string]time.Duration{ - "one": 300 * time.Millisecond, + "one": 200 * time.Millisecond, "two": 100 * time.Millisecond, - "three": 200 * time.Millisecond, - "four": 0 * time.Millisecond, + "three": 0 * time.Millisecond, + "four": 300 * time.Millisecond, }, }, } diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 92aa5fbc157..e1d62dc7a8a 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -24,7 +24,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chainlink-automation v0.8.1 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 @@ -198,7 +198,7 @@ require ( github.com/hashicorp/go-envparse v0.1.0 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 // indirect + github.com/hashicorp/go-plugin v1.6.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 9b0336635bb..9b2fadc06d1 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -648,8 +648,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 h1:OSQYEsRT3tRttZkk6zyC3aAaliwd7Loi/KgXgXxGtwA= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= @@ -1092,8 +1092,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 h1:BdqTkM2KObIaD2vevGM5MVJz+3pZl3wNF8h68Gh5iys= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80/go.mod h1:4adKaHNaxFsRvV/lYfqtbsWyyvIPUMLR0FdOJN/ljis= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 h1:H6i0LEvXB0se/63E3jE9N0/7TugOYLpK4e6TT6a0omc= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595/go.mod h1:TQ9/KKXZ9vr8QAlUquqGpSvDCpR+DtABKPXZY4CiRns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 h1:dO4GoU3YTdnl9c7viiO5qQ12+rpETyfiX7LGsHsa6Ns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3/go.mod h1:ny87uTW6hLjCTLiBqBRNFEhETSXhHWevYlPclT5lSco= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f h1:BwrIaQIx5Iy6eT+DfLhFfK2XqjxRm74mVdlX8gbu4dw= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f/go.mod h1:wHtwSR3F1CQSJJZDQKuqaqFYnvkT+kMyget7dl8Clvo= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e h1:JiETqdNM0bktAUGMc62COwXIaw3rR3M77Me6bBLG0Fg= diff --git a/core/services/registrysyncer/monitoring.go b/core/services/registrysyncer/monitoring.go index 97d139d044f..51d74f22971 100644 --- a/core/services/registrysyncer/monitoring.go +++ b/core/services/registrysyncer/monitoring.go @@ -12,39 +12,38 @@ import ( localMonitoring "github.com/smartcontractkit/chainlink/v2/core/monitoring" ) -var remoteRegistrySyncFailureCounter metric.Int64Counter -var launcherFailureCounter metric.Int64Counter +// syncerMetricLabeler wraps monitoring.MetricsLabeler to provide workflow specific utilities +// for monitoring resources +type syncerMetricLabeler struct { + metrics.Labeler + remoteRegistrySyncFailureCounter metric.Int64Counter + launcherFailureCounter metric.Int64Counter +} -func initMonitoringResources() (err error) { - remoteRegistrySyncFailureCounter, err = beholder.GetMeter().Int64Counter("RemoteRegistrySyncFailure") +func newSyncerMetricLabeler() (*syncerMetricLabeler, error) { + remoteRegistrySyncFailureCounter, err := beholder.GetMeter().Int64Counter("RemoteRegistrySyncFailure") if err != nil { - return fmt.Errorf("failed to register sync failure counter: %w", err) + return nil, fmt.Errorf("failed to register sync failure counter: %w", err) } - launcherFailureCounter, err = beholder.GetMeter().Int64Counter("LauncherFailureCounter") + launcherFailureCounter, err := beholder.GetMeter().Int64Counter("LauncherFailureCounter") if err != nil { - return fmt.Errorf("failed to register launcher failure counter: %w", err) + return nil, fmt.Errorf("failed to register launcher failure counter: %w", err) } - return nil -} - -// syncerMetricLabeler wraps monitoring.MetricsLabeler to provide workflow specific utilities -// for monitoring resources -type syncerMetricLabeler struct { - metrics.Labeler + return &syncerMetricLabeler{remoteRegistrySyncFailureCounter: remoteRegistrySyncFailureCounter, launcherFailureCounter: launcherFailureCounter}, nil } -func (c syncerMetricLabeler) with(keyValues ...string) syncerMetricLabeler { - return syncerMetricLabeler{c.With(keyValues...)} +func (c *syncerMetricLabeler) with(keyValues ...string) syncerMetricLabeler { + return syncerMetricLabeler{c.With(keyValues...), c.remoteRegistrySyncFailureCounter, c.launcherFailureCounter} } -func (c syncerMetricLabeler) incrementRemoteRegistryFailureCounter(ctx context.Context) { +func (c *syncerMetricLabeler) incrementRemoteRegistryFailureCounter(ctx context.Context) { otelLabels := localMonitoring.KvMapToOtelAttributes(c.Labels) - remoteRegistrySyncFailureCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...)) + c.remoteRegistrySyncFailureCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...)) } -func (c syncerMetricLabeler) incrementLauncherFailureCounter(ctx context.Context) { +func (c *syncerMetricLabeler) incrementLauncherFailureCounter(ctx context.Context) { otelLabels := localMonitoring.KvMapToOtelAttributes(c.Labels) - launcherFailureCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...)) + c.launcherFailureCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...)) } diff --git a/core/services/registrysyncer/monitoring_test.go b/core/services/registrysyncer/monitoring_test.go index 1ddb6c57997..30d773aa976 100644 --- a/core/services/registrysyncer/monitoring_test.go +++ b/core/services/registrysyncer/monitoring_test.go @@ -9,11 +9,12 @@ import ( ) func Test_InitMonitoringResources(t *testing.T) { - require.NoError(t, initMonitoringResources()) + _, err := newSyncerMetricLabeler() + require.NoError(t, err) } func Test_SyncerMetricsLabeler(t *testing.T) { - testSyncerMetricLabeler := syncerMetricLabeler{metrics.NewLabeler()} + testSyncerMetricLabeler := syncerMetricLabeler{metrics.NewLabeler(), nil, nil} testSyncerMetricLabeler2 := testSyncerMetricLabeler.with("foo", "baz") require.EqualValues(t, testSyncerMetricLabeler2.Labels["foo"], "baz") } diff --git a/core/services/registrysyncer/syncer.go b/core/services/registrysyncer/syncer.go index 40d105e2b8e..0ef543e95db 100644 --- a/core/services/registrysyncer/syncer.go +++ b/core/services/registrysyncer/syncer.go @@ -44,7 +44,7 @@ type RegistrySyncer interface { type registrySyncer struct { services.StateMachine - metrics syncerMetricLabeler + metrics *syncerMetricLabeler stopCh services.StopChan launchers []Launcher reader types.ContractReader @@ -76,7 +76,14 @@ func New( registryAddress string, orm ORM, ) (RegistrySyncer, error) { + + metricLabeler, err := newSyncerMetricLabeler() + if err != nil { + return nil, fmt.Errorf("failed to create syncer metric labeler: %w", err) + } + return ®istrySyncer{ + metrics: metricLabeler, stopCh: make(services.StopChan), updateChan: make(chan *LocalRegistry), lggr: lggr.Named("RegistrySyncer"), @@ -131,11 +138,6 @@ func newReader(ctx context.Context, lggr logger.Logger, relayer ContractReaderFa func (s *registrySyncer) Start(ctx context.Context) error { return s.StartOnce("RegistrySyncer", func() error { - err := initMonitoringResources() - if err != nil { - return err - } - s.wg.Add(1) go func() { defer s.wg.Done() diff --git a/core/services/relay/evm/chain_reader.go b/core/services/relay/evm/chain_reader.go index c734ade1104..07847b22b4f 100644 --- a/core/services/relay/evm/chain_reader.go +++ b/core/services/relay/evm/chain_reader.go @@ -6,6 +6,7 @@ import ( "fmt" "maps" "slices" + "strconv" "strings" "time" @@ -219,6 +220,41 @@ func (cr *chainReader) GetLatestValue(ctx context.Context, readName string, conf return nil } +func (cr *chainReader) GetLatestValueWithHeadData(ctx context.Context, readName string, confidenceLevel primitives.ConfidenceLevel, params any, returnVal any) (*commontypes.Head, error) { + if err := cr.GetLatestValue(ctx, readName, confidenceLevel, params, returnVal); err != nil { + return nil, err + } + + latestHead, finalizedHead, err := cr.ht.LatestAndFinalizedBlock(ctx) + if err != nil { + return nil, err + } + + switch confidenceLevel { + case primitives.Finalized: + if finalizedHead != nil { + return ethHeadToCommonHead(finalizedHead), nil + } + return nil, nil + case primitives.Unconfirmed: + if latestHead != nil { + return ethHeadToCommonHead(latestHead), nil + } + return nil, nil + default: + return nil, errors.New("unsupported confidence level") + } +} + +func ethHeadToCommonHead(finalizedHead *evmtypes.Head) *commontypes.Head { + return &commontypes.Head{ + Height: strconv.FormatInt(finalizedHead.Number, 10), + Hash: finalizedHead.Hash.Bytes(), + //nolint:gosec // disable G115 + Timestamp: uint64(finalizedHead.Timestamp.Unix()), + } +} + func (cr *chainReader) BatchGetLatestValues(ctx context.Context, request commontypes.BatchGetLatestValuesRequest) (commontypes.BatchGetLatestValuesResult, error) { return cr.bindings.BatchGetLatestValues(ctx, request) } diff --git a/core/services/workflows/engine.go b/core/services/workflows/engine.go index a814b6a9007..8133d98611c 100644 --- a/core/services/workflows/engine.go +++ b/core/services/workflows/engine.go @@ -276,6 +276,7 @@ func (e *Engine) initializeCapability(ctx context.Context, step *step) error { Metadata: capabilities.RegistrationMetadata{ WorkflowID: e.workflow.id, WorkflowOwner: e.workflow.owner, + ReferenceID: step.Vertex.Ref, }, Config: stepConfig, } @@ -1065,6 +1066,7 @@ func (e *Engine) Close() error { Metadata: capabilities.RegistrationMetadata{ WorkflowID: e.workflow.id, WorkflowOwner: e.workflow.owner, + ReferenceID: s.Vertex.Ref, }, Config: stepConfig, } diff --git a/deployment/go.mod b/deployment/go.mod index f73b5cc6b4d..de92c699962 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -24,7 +24,7 @@ require ( github.com/smartcontractkit/ccip-owner-contracts v0.0.0-20240926212305-a6deabdfce86 github.com/smartcontractkit/chain-selectors v1.0.27 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 github.com/smartcontractkit/chainlink-protos/job-distributor v0.4.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.12 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 @@ -264,7 +264,7 @@ require ( github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 // indirect + github.com/hashicorp/go-plugin v1.6.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-sockaddr v1.0.6 // indirect diff --git a/deployment/go.sum b/deployment/go.sum index e854710fb5a..0e62f62b912 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -849,8 +849,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 h1:OSQYEsRT3tRttZkk6zyC3aAaliwd7Loi/KgXgXxGtwA= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= @@ -1387,8 +1387,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 h1:BdqTkM2KObIaD2vevGM5MVJz+3pZl3wNF8h68Gh5iys= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80/go.mod h1:4adKaHNaxFsRvV/lYfqtbsWyyvIPUMLR0FdOJN/ljis= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 h1:H6i0LEvXB0se/63E3jE9N0/7TugOYLpK4e6TT6a0omc= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595/go.mod h1:TQ9/KKXZ9vr8QAlUquqGpSvDCpR+DtABKPXZY4CiRns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 h1:dO4GoU3YTdnl9c7viiO5qQ12+rpETyfiX7LGsHsa6Ns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3/go.mod h1:ny87uTW6hLjCTLiBqBRNFEhETSXhHWevYlPclT5lSco= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f h1:BwrIaQIx5Iy6eT+DfLhFfK2XqjxRm74mVdlX8gbu4dw= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f/go.mod h1:wHtwSR3F1CQSJJZDQKuqaqFYnvkT+kMyget7dl8Clvo= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e h1:JiETqdNM0bktAUGMc62COwXIaw3rR3M77Me6bBLG0Fg= diff --git a/go.mod b/go.mod index 7b41a80c823..3ea16198409 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/graph-gophers/graphql-go v1.5.0 github.com/hashicorp/consul/sdk v0.16.0 github.com/hashicorp/go-envparse v0.1.0 - github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 + github.com/hashicorp/go-plugin v1.6.2 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/hdevalence/ed25519consensus v0.1.0 github.com/jackc/pgconn v1.14.3 @@ -76,7 +76,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.27 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e github.com/smartcontractkit/chainlink-feeds v0.1.1 diff --git a/go.sum b/go.sum index ce3338b4b6c..0fb8e852398 100644 --- a/go.sum +++ b/go.sum @@ -636,8 +636,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 h1:OSQYEsRT3tRttZkk6zyC3aAaliwd7Loi/KgXgXxGtwA= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= @@ -1077,8 +1077,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 h1:BdqTkM2KObIaD2vevGM5MVJz+3pZl3wNF8h68Gh5iys= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80/go.mod h1:4adKaHNaxFsRvV/lYfqtbsWyyvIPUMLR0FdOJN/ljis= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 h1:H6i0LEvXB0se/63E3jE9N0/7TugOYLpK4e6TT6a0omc= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595/go.mod h1:TQ9/KKXZ9vr8QAlUquqGpSvDCpR+DtABKPXZY4CiRns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 h1:dO4GoU3YTdnl9c7viiO5qQ12+rpETyfiX7LGsHsa6Ns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3/go.mod h1:ny87uTW6hLjCTLiBqBRNFEhETSXhHWevYlPclT5lSco= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f h1:BwrIaQIx5Iy6eT+DfLhFfK2XqjxRm74mVdlX8gbu4dw= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f/go.mod h1:wHtwSR3F1CQSJJZDQKuqaqFYnvkT+kMyget7dl8Clvo= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e h1:JiETqdNM0bktAUGMc62COwXIaw3rR3M77Me6bBLG0Fg= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index eec51bbca41..7372cfc637a 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -36,7 +36,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.27 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 github.com/smartcontractkit/chainlink-protos/job-distributor v0.4.0 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 @@ -285,7 +285,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 // indirect + github.com/hashicorp/go-plugin v1.6.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-sockaddr v1.0.6 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index e96918be689..b80fb5d5b4e 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -855,8 +855,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 h1:OSQYEsRT3tRttZkk6zyC3aAaliwd7Loi/KgXgXxGtwA= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= @@ -1405,8 +1405,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 h1:BdqTkM2KObIaD2vevGM5MVJz+3pZl3wNF8h68Gh5iys= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80/go.mod h1:4adKaHNaxFsRvV/lYfqtbsWyyvIPUMLR0FdOJN/ljis= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 h1:H6i0LEvXB0se/63E3jE9N0/7TugOYLpK4e6TT6a0omc= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595/go.mod h1:TQ9/KKXZ9vr8QAlUquqGpSvDCpR+DtABKPXZY4CiRns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 h1:dO4GoU3YTdnl9c7viiO5qQ12+rpETyfiX7LGsHsa6Ns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3/go.mod h1:ny87uTW6hLjCTLiBqBRNFEhETSXhHWevYlPclT5lSco= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f h1:BwrIaQIx5Iy6eT+DfLhFfK2XqjxRm74mVdlX8gbu4dw= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f/go.mod h1:wHtwSR3F1CQSJJZDQKuqaqFYnvkT+kMyget7dl8Clvo= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e h1:JiETqdNM0bktAUGMc62COwXIaw3rR3M77Me6bBLG0Fg= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index b85078806cf..0bc7d3233bc 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 github.com/slack-go/slack v0.15.0 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 @@ -289,7 +289,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 // indirect + github.com/hashicorp/go-plugin v1.6.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-sockaddr v1.0.6 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index a075600dcbc..7ede075270a 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -853,8 +853,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 h1:OSQYEsRT3tRttZkk6zyC3aAaliwd7Loi/KgXgXxGtwA= -github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= @@ -1394,8 +1394,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80 h1:BdqTkM2KObIaD2vevGM5MVJz+3pZl3wNF8h68Gh5iys= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241029124258-09d605658a80/go.mod h1:4adKaHNaxFsRvV/lYfqtbsWyyvIPUMLR0FdOJN/ljis= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595 h1:H6i0LEvXB0se/63E3jE9N0/7TugOYLpK4e6TT6a0omc= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241025132045-cfad02139595/go.mod h1:TQ9/KKXZ9vr8QAlUquqGpSvDCpR+DtABKPXZY4CiRns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3 h1:dO4GoU3YTdnl9c7viiO5qQ12+rpETyfiX7LGsHsa6Ns= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241111100612-38cd8cbf86f3/go.mod h1:ny87uTW6hLjCTLiBqBRNFEhETSXhHWevYlPclT5lSco= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f h1:BwrIaQIx5Iy6eT+DfLhFfK2XqjxRm74mVdlX8gbu4dw= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f/go.mod h1:wHtwSR3F1CQSJJZDQKuqaqFYnvkT+kMyget7dl8Clvo= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e h1:JiETqdNM0bktAUGMc62COwXIaw3rR3M77Me6bBLG0Fg=