From de23cc284b0585bb6259fc66d9a42913f4eab50f Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 11 Jun 2024 17:02:06 -0500 Subject: [PATCH 1/8] Add GAS opcode to Stylus tracing, and after producing a fake GAS opcode in a trace, add a POP afterwards --- arbos/programs/api.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/arbos/programs/api.go b/arbos/programs/api.go index c8241a72b5..d803e93435 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -267,6 +267,12 @@ func newApiClosures( captureHostio := func(name string, args, outs []byte, startInk, endInk uint64) { tracingInfo.Tracer.CaptureStylusHostio(name, args, outs, startInk, endInk) } + recordGasOpcode := func() { + if tracingInfo != nil { + tracingInfo.Tracer.CaptureState(0, vm.GAS, 0, 0, scope, []byte{}, depth, nil) + tracingInfo.Tracer.CaptureState(0, vm.POP, 0, 0, scope, []byte{}, depth, nil) + } + } return func(req RequestType, input []byte) ([]byte, []byte, uint64) { original := input @@ -314,6 +320,12 @@ func newApiClosures( input = []byte{} return data } + takeGasLeft := func() uint64 { + defer func() { + recordGasOpcode() + }() + return takeU64() + } switch req { case GetBytes32: @@ -321,7 +333,7 @@ func newApiClosures( out, cost := getBytes32(key) return out[:], nil, cost case SetTrieSlots: - gasLeft := takeU64() + gasLeft := takeGasLeft() gas := gasLeft status := setTrieSlots(takeRest(), &gas) return status.to_slice(), nil, gasLeft - gas @@ -348,7 +360,7 @@ func newApiClosures( } contract := takeAddress() value := takeU256() - gasLeft := takeU64() + gasLeft := takeGasLeft() gasReq := takeU64() calldata := takeRest() @@ -359,7 +371,7 @@ func newApiClosures( } return []byte{statusByte}, ret, cost case Create1, Create2: - gas := takeU64() + gas := takeGasLeft() endowment := takeU256() var salt *u256 if req == Create2 { @@ -392,7 +404,7 @@ func newApiClosures( return balance[:], nil, cost case AccountCode: address := takeAddress() - gas := takeU64() + gas := takeGasLeft() code, cost := accountCode(address, gas) return nil, code, cost case AccountCodeHash: From c1033490c7a67f8dc5d9184dceb55499850c84fc Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 12 Jun 2024 10:34:19 -0500 Subject: [PATCH 2/8] fix impl --- arbos/programs/api.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/arbos/programs/api.go b/arbos/programs/api.go index d803e93435..787f127ea4 100644 --- a/arbos/programs/api.go +++ b/arbos/programs/api.go @@ -266,9 +266,7 @@ func newApiClosures( } captureHostio := func(name string, args, outs []byte, startInk, endInk uint64) { tracingInfo.Tracer.CaptureStylusHostio(name, args, outs, startInk, endInk) - } - recordGasOpcode := func() { - if tracingInfo != nil { + if name == "evm_gas_left" || name == "evm_ink_left" { tracingInfo.Tracer.CaptureState(0, vm.GAS, 0, 0, scope, []byte{}, depth, nil) tracingInfo.Tracer.CaptureState(0, vm.POP, 0, 0, scope, []byte{}, depth, nil) } @@ -320,12 +318,6 @@ func newApiClosures( input = []byte{} return data } - takeGasLeft := func() uint64 { - defer func() { - recordGasOpcode() - }() - return takeU64() - } switch req { case GetBytes32: @@ -333,7 +325,7 @@ func newApiClosures( out, cost := getBytes32(key) return out[:], nil, cost case SetTrieSlots: - gasLeft := takeGasLeft() + gasLeft := takeU64() gas := gasLeft status := setTrieSlots(takeRest(), &gas) return status.to_slice(), nil, gasLeft - gas @@ -360,7 +352,7 @@ func newApiClosures( } contract := takeAddress() value := takeU256() - gasLeft := takeGasLeft() + gasLeft := takeU64() gasReq := takeU64() calldata := takeRest() @@ -371,7 +363,7 @@ func newApiClosures( } return []byte{statusByte}, ret, cost case Create1, Create2: - gas := takeGasLeft() + gas := takeU64() endowment := takeU256() var salt *u256 if req == Create2 { @@ -404,7 +396,7 @@ func newApiClosures( return balance[:], nil, cost case AccountCode: address := takeAddress() - gas := takeGasLeft() + gas := takeU64() code, cost := accountCode(address, gas) return nil, code, cost case AccountCodeHash: From ee94f901bff6d4a6f33a46130eaa27c17adb7efc Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 14 Jun 2024 16:21:19 -0500 Subject: [PATCH 3/8] Add iostat system metrics --- Dockerfile | 3 +- cmd/nitro/nitro.go | 10 ++++ util/iostat/iostat.go | 132 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 util/iostat/iostat.go diff --git a/Dockerfile b/Dockerfile index 329d57c0a6..4f5d378fc5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -255,7 +255,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ ca-certificates \ - wabt && \ + wabt \ + sysstat && \ /usr/sbin/update-ca-certificates && \ useradd -s /bin/bash user && \ mkdir -p /home/user/l1keystore && \ diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index e194a8e837..9d73f58406 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -60,6 +60,7 @@ import ( "github.com/offchainlabs/nitro/staker/validatorwallet" "github.com/offchainlabs/nitro/util/colors" "github.com/offchainlabs/nitro/util/headerreader" + "github.com/offchainlabs/nitro/util/iostat" "github.com/offchainlabs/nitro/util/rpcclient" "github.com/offchainlabs/nitro/util/signature" "github.com/offchainlabs/nitro/validator/server_common" @@ -404,6 +405,15 @@ func mainImpl() int { return 1 } + if nodeConfig.Metrics { + iostatMetrics := iostat.NewMetricsSpawner() + if err := iostatMetrics.RegisterMetrics(ctx, 1); err != nil { + log.Error("Error registering iostat metrics, disabling them", "err", err) + } else { + go iostatMetrics.PopulateMetrics() + } + } + var deferFuncs []func() defer func() { for i := range deferFuncs { diff --git a/util/iostat/iostat.go b/util/iostat/iostat.go new file mode 100644 index 0000000000..e98f42de97 --- /dev/null +++ b/util/iostat/iostat.go @@ -0,0 +1,132 @@ +package iostat + +import ( + "bufio" + "context" + "errors" + "fmt" + "os/exec" + "strconv" + "strings" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +type MetricsSpawner struct { + statReceiver chan DeviceStats + metrics map[string]map[string]metrics.GaugeFloat64 +} + +func NewMetricsSpawner() *MetricsSpawner { + return &MetricsSpawner{ + metrics: make(map[string]map[string]metrics.GaugeFloat64), + statReceiver: make(chan DeviceStats), + } +} + +func (m *MetricsSpawner) RegisterMetrics(ctx context.Context, spwanInterval int) error { + go Run(ctx, spwanInterval, m.statReceiver) + // Register metrics for a maximum of 5 devices (fail safe incase iostat command returns incorrect names indefinitely) + for i := 0; i < 5; i++ { + stat, ok := <-m.statReceiver + if !ok { + return errors.New("failed to register iostat metrics") + } + if _, ok := m.metrics[stat.DeviceName]; ok { + return nil + } + baseMetricName := fmt.Sprintf("isotat/%s/", stat.DeviceName) + m.metrics[stat.DeviceName] = make(map[string]metrics.GaugeFloat64) + m.metrics[stat.DeviceName]["readspersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"readspersecond", nil) + m.metrics[stat.DeviceName]["writespersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"writespersecond", nil) + m.metrics[stat.DeviceName]["await"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"await", nil) + } + return nil +} + +func (m *MetricsSpawner) PopulateMetrics() { + for { + stat, ok := <-m.statReceiver + if !ok { + log.Info("Iostat statReceiver channel was closed due to error or command being completed") + return + } + if _, ok := m.metrics[stat.DeviceName]; !ok { + log.Warn("Unrecognized device name in output of iostat command", "deviceName", stat.DeviceName) + continue + } + m.metrics[stat.DeviceName]["readspersecond"].Update(stat.ReadsPerSecond) + m.metrics[stat.DeviceName]["writespersecond"].Update(stat.WritesPerSecond) + m.metrics[stat.DeviceName]["await"].Update(stat.Await) + } +} + +type DeviceStats struct { + DeviceName string + ReadsPerSecond float64 + WritesPerSecond float64 + Await float64 +} + +func Run(ctx context.Context, interval int, receiver chan DeviceStats) { + defer close(receiver) + // #nosec G204 + cmd := exec.Command("iostat", "-dNxy", strconv.Itoa(interval)) + stdout, err := cmd.StdoutPipe() + if err != nil { + log.Error("Failed to get stdout", "err", err) + return + } + if err := cmd.Start(); err != nil { + log.Error("Failed to start iostat command", "err", err) + return + } + var fields []string + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + if ctx.Err() != nil { + log.Error("Context error when running iostat metrics", "err", ctx.Err()) + return + } + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "Device") { + fields = strings.Fields(line) + continue + } + data := strings.Fields(line) + if len(data) == 0 { + continue + } + stat := DeviceStats{} + var err error + for i, field := range fields { + switch field { + case "Device", "Device:": + stat.DeviceName = data[i] + case "r/s": + stat.ReadsPerSecond, err = strconv.ParseFloat(data[i], 64) + case "w/s": + stat.WritesPerSecond, err = strconv.ParseFloat(data[i], 64) + case "await": + stat.Await, err = strconv.ParseFloat(data[i], 64) + } + if err != nil { + log.Error("Error parsing command result from iostat", "err", err) + continue + } + } + if stat.DeviceName == "" { + continue + } + receiver <- stat + } + if err := cmd.Process.Kill(); err != nil { + log.Error("Failed to kill iostat process", "err", err) + } + if err := cmd.Wait(); err != nil { + log.Error("Error waiting for iostat to exit", "err", err) + } + stdout.Close() + log.Info("Iostat command terminated") +} From 97d3da080ffd50a6f748bc0c654fb32c1a55d1e6 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 17 Jun 2024 14:42:30 -0500 Subject: [PATCH 4/8] address PR comments --- cmd/nitro/nitro.go | 7 +---- util/iostat/iostat.go | 72 ++++++++++++++++--------------------------- 2 files changed, 28 insertions(+), 51 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 9d73f58406..46382d29d1 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -406,12 +406,7 @@ func mainImpl() int { } if nodeConfig.Metrics { - iostatMetrics := iostat.NewMetricsSpawner() - if err := iostatMetrics.RegisterMetrics(ctx, 1); err != nil { - log.Error("Error registering iostat metrics, disabling them", "err", err) - } else { - go iostatMetrics.PopulateMetrics() - } + go iostat.RegisterAndPopulateMetrics(ctx, 1, 5) } var deferFuncs []func() diff --git a/util/iostat/iostat.go b/util/iostat/iostat.go index e98f42de97..353c0c1f88 100644 --- a/util/iostat/iostat.go +++ b/util/iostat/iostat.go @@ -3,9 +3,9 @@ package iostat import ( "bufio" "context" - "errors" "fmt" "os/exec" + "runtime" "strconv" "strings" @@ -13,52 +13,35 @@ import ( "github.com/ethereum/go-ethereum/metrics" ) -type MetricsSpawner struct { - statReceiver chan DeviceStats - metrics map[string]map[string]metrics.GaugeFloat64 -} - -func NewMetricsSpawner() *MetricsSpawner { - return &MetricsSpawner{ - metrics: make(map[string]map[string]metrics.GaugeFloat64), - statReceiver: make(chan DeviceStats), - } -} - -func (m *MetricsSpawner) RegisterMetrics(ctx context.Context, spwanInterval int) error { - go Run(ctx, spwanInterval, m.statReceiver) - // Register metrics for a maximum of 5 devices (fail safe incase iostat command returns incorrect names indefinitely) - for i := 0; i < 5; i++ { - stat, ok := <-m.statReceiver - if !ok { - return errors.New("failed to register iostat metrics") - } - if _, ok := m.metrics[stat.DeviceName]; ok { - return nil - } - baseMetricName := fmt.Sprintf("isotat/%s/", stat.DeviceName) - m.metrics[stat.DeviceName] = make(map[string]metrics.GaugeFloat64) - m.metrics[stat.DeviceName]["readspersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"readspersecond", nil) - m.metrics[stat.DeviceName]["writespersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"writespersecond", nil) - m.metrics[stat.DeviceName]["await"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"await", nil) +func RegisterAndPopulateMetrics(ctx context.Context, spawnInterval, maxDeviceCount int) { + deviceMetrics := make(map[string]map[string]metrics.GaugeFloat64) + statReceiver := make(chan DeviceStats) + if runtime.GOOS != "linux" { + log.Warn("Iostat command not supported disabling corresponding metrics") + return } - return nil -} - -func (m *MetricsSpawner) PopulateMetrics() { + go Run(ctx, spawnInterval, statReceiver) for { - stat, ok := <-m.statReceiver + stat, ok := <-statReceiver if !ok { log.Info("Iostat statReceiver channel was closed due to error or command being completed") return } - if _, ok := m.metrics[stat.DeviceName]; !ok { - log.Warn("Unrecognized device name in output of iostat command", "deviceName", stat.DeviceName) - continue + if _, ok := deviceMetrics[stat.DeviceName]; !ok { + // Register metrics for a maximum of maxDeviceCount (fail safe incase iostat command returns incorrect names indefinitely) + if len(deviceMetrics) < maxDeviceCount { + baseMetricName := fmt.Sprintf("isotat/%s/", stat.DeviceName) + deviceMetrics[stat.DeviceName] = make(map[string]metrics.GaugeFloat64) + deviceMetrics[stat.DeviceName]["readspersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"readspersecond", nil) + deviceMetrics[stat.DeviceName]["writespersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"writespersecond", nil) + deviceMetrics[stat.DeviceName]["await"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"await", nil) + } else { + continue + } } - m.metrics[stat.DeviceName]["readspersecond"].Update(stat.ReadsPerSecond) - m.metrics[stat.DeviceName]["writespersecond"].Update(stat.WritesPerSecond) - m.metrics[stat.DeviceName]["await"].Update(stat.Await) + deviceMetrics[stat.DeviceName]["readspersecond"].Update(stat.ReadsPerSecond) + deviceMetrics[stat.DeviceName]["writespersecond"].Update(stat.WritesPerSecond) + deviceMetrics[stat.DeviceName]["await"].Update(stat.Await) } } @@ -72,7 +55,7 @@ type DeviceStats struct { func Run(ctx context.Context, interval int, receiver chan DeviceStats) { defer close(receiver) // #nosec G204 - cmd := exec.Command("iostat", "-dNxy", strconv.Itoa(interval)) + cmd := exec.CommandContext(ctx, "iostat", "-dNxy", strconv.Itoa(interval)) stdout, err := cmd.StdoutPipe() if err != nil { log.Error("Failed to get stdout", "err", err) @@ -85,10 +68,6 @@ func Run(ctx context.Context, interval int, receiver chan DeviceStats) { var fields []string scanner := bufio.NewScanner(stdout) for scanner.Scan() { - if ctx.Err() != nil { - log.Error("Context error when running iostat metrics", "err", ctx.Err()) - return - } line := strings.TrimSpace(scanner.Text()) if strings.HasPrefix(line, "Device") { fields = strings.Fields(line) @@ -121,6 +100,9 @@ func Run(ctx context.Context, interval int, receiver chan DeviceStats) { } receiver <- stat } + if scanner.Err() != nil { + log.Error("Iostat scanner error", err, scanner.Err()) + } if err := cmd.Process.Kill(); err != nil { log.Error("Failed to kill iostat process", "err", err) } From 7ebfa5aacc89c263acd68df645c97f91bfe333e8 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 17 Jun 2024 14:53:43 -0500 Subject: [PATCH 5/8] code refactor --- util/iostat/iostat.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/iostat/iostat.go b/util/iostat/iostat.go index 353c0c1f88..9bc5ff800c 100644 --- a/util/iostat/iostat.go +++ b/util/iostat/iostat.go @@ -14,12 +14,12 @@ import ( ) func RegisterAndPopulateMetrics(ctx context.Context, spawnInterval, maxDeviceCount int) { - deviceMetrics := make(map[string]map[string]metrics.GaugeFloat64) - statReceiver := make(chan DeviceStats) if runtime.GOOS != "linux" { log.Warn("Iostat command not supported disabling corresponding metrics") return } + deviceMetrics := make(map[string]map[string]metrics.GaugeFloat64) + statReceiver := make(chan DeviceStats) go Run(ctx, spawnInterval, statReceiver) for { stat, ok := <-statReceiver From 3b5b22efa6beb8f3309eb058b82aa2a27301e50a Mon Sep 17 00:00:00 2001 From: Tristan Wilson Date: Tue, 18 Jun 2024 13:16:39 -0700 Subject: [PATCH 6/8] Metric for any das.Aggregator Store errors If there are errors sending to any backends in das.Aggregator.Store then the new metric arb_das_rpc_aggregator_store_error_gauge will be 1 until there is a call to Store with no errors. The metric will be 1 even if the Store succeeds, as long as at least one backend failed. --- das/aggregator.go | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/das/aggregator.go b/das/aggregator.go index 25db73a76e..181df67783 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "math/bits" + "sync/atomic" "time" flag "github.com/spf13/pflag" @@ -26,6 +27,16 @@ import ( "github.com/offchainlabs/nitro/util/pretty" ) +const metricBase string = "arb/das/rpc/aggregator/store" + +var ( + // This metric shows 1 if there was any error posting to the backends, until + // there was a Store that had no backend failures. + anyErrorGauge = metrics.GetOrRegisterGauge(metricBase+"/error/gauge", nil) + +// Other aggregator metrics are generated dynamically in the Store function. +) + type AggregatorConfig struct { Enable bool `koanf:"enable"` AssumedHonest int `koanf:"assumed-honest"` @@ -167,6 +178,16 @@ type storeResponse struct { // signature is not checked, which is useful for testing. func (a *Aggregator) Store(ctx context.Context, message []byte, timeout uint64, sig []byte) (*daprovider.DataAvailabilityCertificate, error) { log.Trace("das.Aggregator.Store", "message", pretty.FirstFewBytes(message), "timeout", time.Unix(int64(timeout), 0), "sig", pretty.FirstFewBytes(sig)) + + allBackendsSucceeded := false + defer func() { + if allBackendsSucceeded { + anyErrorGauge.Update(0) + } else { + anyErrorGauge.Update(1) + } + }() + if a.addrVerifier != nil { actualSigner, err := DasRecoverSigner(message, sig, timeout) if err != nil { @@ -187,7 +208,6 @@ func (a *Aggregator) Store(ctx context.Context, message []byte, timeout uint64, for _, d := range a.services { go func(ctx context.Context, d ServiceDetails) { storeCtx, cancel := context.WithTimeout(ctx, a.requestTimeout) - const metricBase string = "arb/das/rpc/aggregator/store" var metricWithServiceName = metricBase + "/" + d.metricName defer cancel() incFailureMetric := func() { @@ -253,22 +273,22 @@ func (a *Aggregator) Store(ctx context.Context, message []byte, timeout uint64, err error } + var storeFailures atomic.Int64 // Collect responses from backends. certDetailsChan := make(chan certDetails) go func() { var pubKeys []blsSignatures.PublicKey var sigs []blsSignatures.Signature var aggSignersMask uint64 - var storeFailures, successfullyStoredCount int + var successfullyStoredCount int var returned bool for i := 0; i < len(a.services); i++ { - select { case <-ctx.Done(): break case r := <-responses: if r.err != nil { - storeFailures++ + _ = storeFailures.Add(1) log.Warn("das.Aggregator: Error from backend", "backend", r.details.service, "signerMask", r.details.signersMask, "err", r.err) } else { pubKeys = append(pubKeys, r.details.pubKey) @@ -292,10 +312,10 @@ func (a *Aggregator) Store(ctx context.Context, message []byte, timeout uint64, certDetailsChan <- cd returned = true if a.maxAllowedServiceStoreFailures > 0 && // Ignore the case where AssumedHonest = 1, probably a testnet - storeFailures+1 > a.maxAllowedServiceStoreFailures { + int(storeFailures.Load())+1 > a.maxAllowedServiceStoreFailures { log.Error("das.Aggregator: storing the batch data succeeded to enough DAS commitee members to generate the Data Availability Cert, but if one more had failed then the cert would not have been able to be generated. Look for preceding logs with \"Error from backend\"") } - } else if storeFailures > a.maxAllowedServiceStoreFailures { + } else if int(storeFailures.Load()) > a.maxAllowedServiceStoreFailures { cd := certDetails{} cd.err = fmt.Errorf("aggregator failed to store message to at least %d out of %d DASes (assuming %d are honest). %w", a.requiredServicesForStore, len(a.services), a.config.AssumedHonest, daprovider.ErrBatchToDasFailed) certDetailsChan <- cd @@ -329,6 +349,11 @@ func (a *Aggregator) Store(ctx context.Context, message []byte, timeout uint64, if !verified { return nil, fmt.Errorf("failed aggregate signature check. %w", daprovider.ErrBatchToDasFailed) } + + if storeFailures.Load() == 0 { + allBackendsSucceeded = true + } + return &aggCert, nil } From c12543ff42cac061dbea2b74f5a9d353b02857af Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 26 Jun 2024 19:54:06 -0600 Subject: [PATCH 7/8] build arbitrator.h on test-go-deps --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 53b89c8d72..dc9b4e3ddf 100644 --- a/Makefile +++ b/Makefile @@ -162,6 +162,7 @@ test-go-deps: \ build-replay-env \ $(stylus_test_wasms) \ $(arbitrator_stylus_lib) \ + $(arbitrator_generated_header) \ $(patsubst %,$(arbitrator_cases)/%.wasm, global-state read-inboxmsg-10 global-state-wrapper const) build-prover-header: $(arbitrator_generated_header) From 13495543a609541e5126ea79c6d601c0446fe82e Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 26 Jun 2024 19:57:58 -0600 Subject: [PATCH 8/8] update testnode pin --- nitro-testnode | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitro-testnode b/nitro-testnode index c334820b2d..9dc0588c50 160000 --- a/nitro-testnode +++ b/nitro-testnode @@ -1 +1 @@ -Subproject commit c334820b2dba6dfa4078f81ed242afbbccc19c91 +Subproject commit 9dc0588c5066e2efd25c09adf12df7b28ef18cb6