From 870570724521fe25c0401e27bc1ca9fd8f28bb30 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 3 Nov 2023 10:48:00 -0600 Subject: [PATCH 1/2] rpcclient: add retrydelay --- util/rpcclient/rpcclient.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/util/rpcclient/rpcclient.go b/util/rpcclient/rpcclient.go index 0a09459070..61c125eaed 100644 --- a/util/rpcclient/rpcclient.go +++ b/util/rpcclient/rpcclient.go @@ -28,6 +28,7 @@ type ClientConfig struct { ConnectionWait time.Duration `koanf:"connection-wait"` ArgLogLimit uint `koanf:"arg-log-limit" reload:"hot"` RetryErrors string `koanf:"retry-errors" reload:"hot"` + RetryDelay time.Duration `koanf:"retry-delay"` retryErrors *regexp.Regexp } @@ -63,6 +64,7 @@ func RPCClientAddOptions(prefix string, f *flag.FlagSet, defaultConfig *ClientCo f.Uint(prefix+".arg-log-limit", defaultConfig.ArgLogLimit, "limit size of arguments in log entries") f.Uint(prefix+".retries", defaultConfig.Retries, "number of retries in case of failure(0 mean one attempt)") f.String(prefix+".retry-errors", defaultConfig.RetryErrors, "Errors matching this regular expression are automatically retried") + f.Duration(prefix+".retry-delay", defaultConfig.RetryDelay, "delay between retries") } type RpcClient struct { @@ -131,6 +133,14 @@ func (c *RpcClient) CallContext(ctx_in context.Context, result interface{}, meth log.Trace("sending RPC request", "method", method, "logId", logId, "args", limitedArgumentsMarshal{int(c.config().ArgLogLimit), args}) var err error for i := 0; i < int(c.config().Retries)+1; i++ { + retryDelay := c.config().RetryDelay + if i > 0 && retryDelay > 0 { + select { + case <-ctx_in.Done(): + return ctx_in.Err() + case <-time.After(retryDelay): + } + } if ctx_in.Err() != nil { return ctx_in.Err() } From a6fddc1e6361a61107b91088cc5ac8f94d023961 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 3 Nov 2023 11:00:48 -0600 Subject: [PATCH 2/2] rpcclient: initiaql toxiproxy test --- go.mod | 1 + go.sum | 8 ++ util/rpcclient/rpcclient_test.go | 5 + util/rpcclient/rpcclient_toxiproxy_test.go | 112 +++++++++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 util/rpcclient/rpcclient_toxiproxy_test.go diff --git a/go.mod b/go.mod index 5ab2eda5c7..131d88a0ba 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/DataDog/zstd v1.5.2 // indirect + github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect diff --git a/go.sum b/go.sum index 424e3508c3..07852c74fb 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.30.0/go.mod h1:zujlQQx1kzHsh4jfV1USnptCQrHAEZ2Hk8fTKCulPVs= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae/go.mod h1:/cvHQkZ1fst0EmZnA5dFtiQdWCNCFYzb+uE2vqVgvx0= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= @@ -227,6 +228,7 @@ github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoG github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codeclysm/extract/v3 v3.0.2 h1:sB4LcE3Php7LkhZwN0n2p8GCwZe92PEQutdbGURf5xc= github.com/codeclysm/extract/v3 v3.0.2/go.mod h1:NKsw+hqua9H+Rlwy/w/3Qgt9jDonYEgB6wJu+25eOKw= @@ -265,6 +267,7 @@ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6Uh github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= @@ -380,6 +383,7 @@ github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -871,6 +875,7 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -1245,6 +1250,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= @@ -1386,6 +1392,7 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -1620,6 +1627,7 @@ github.com/wealdtech/go-merkletree v1.0.0/go.mod h1:cdil512d/8ZC7Kx3bfrDvGMQXB25 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= diff --git a/util/rpcclient/rpcclient_test.go b/util/rpcclient/rpcclient_test.go index c9d813ab0d..b885770f60 100644 --- a/util/rpcclient/rpcclient_test.go +++ b/util/rpcclient/rpcclient_test.go @@ -88,6 +88,11 @@ func (t *testAPI) FailAtFirst(ctx context.Context) error { return errors.New("error") } +func (t *testAPI) Delay(ctx context.Context, msec int64) error { + <-time.After(time.Millisecond * time.Duration(msec)) + return nil +} + func TestRpcClientRetry(t *testing.T) { t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) diff --git a/util/rpcclient/rpcclient_toxiproxy_test.go b/util/rpcclient/rpcclient_toxiproxy_test.go new file mode 100644 index 0000000000..31385d50d8 --- /dev/null +++ b/util/rpcclient/rpcclient_toxiproxy_test.go @@ -0,0 +1,112 @@ +//go:build toxiproxy +// +build toxiproxy + +package rpcclient + +import ( + "context" + "testing" + "time" + + toxiproxy "github.com/Shopify/toxiproxy/client" +) + +func TestToxiRpcClient(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) + defer cancel() + + server1 := createTestNode(t, ctx, 0) + + toxiprox := toxiproxy.NewClient("localhost:8474") + proxy, err := toxiprox.CreateProxy("testRpc", "", server1.WSEndpoint()[5:]) + Require(t, err) + defer proxy.Delete() + + config := &ClientConfig{ + URL: "ws://" + proxy.Listen, + Timeout: time.Second * 5, + Retries: 3, + RetryErrors: "websocket: close.*|.* i/o timeout|.*connection reset by peer|dial tcp .*", + RetryDelay: time.Millisecond * 500, + } + Require(t, config.Validate()) + configFetcher := func() *ClientConfig { return config } + + client := NewRpcClient(configFetcher, server1) + + err = client.Start(ctx) + Require(t, err) + + err = client.CallContext(ctx, nil, "test_delay", 500) + Require(t, err) + + errChan := make(chan error) + proxyErrChan := make(chan error) + callDealy := func() { + callCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + errChan <- client.CallContext(callCtx, nil, "test_delay", 3000) + cancel() + } + proxyDisable := func() { + <-time.After(time.Millisecond * 30) + err = proxy.Disable() + if err != nil { + proxyErrChan <- err + return + } + <-time.After(time.Millisecond * 500) + err = proxy.Enable() + proxyErrChan <- err + } + proxyReset := func() { + <-time.After(time.Millisecond * 20) + _, err = proxy.AddToxic("reset_all", "reset_peer", "downstream", 1.0, toxiproxy.Attributes{"timeout": 5}) + if err != nil { + proxyErrChan <- err + return + } + <-time.After(time.Millisecond * 3000) + err = proxy.RemoveToxic("reset_all") + proxyErrChan <- err + } + + config.Retries = 0 + go callDealy() + go proxyDisable() + err = <-proxyErrChan + Require(t, err) + err = <-errChan + if err == nil { + Fail(t, "call during proxyDisable succeeded without retries") + } + + config.Retries = 3 + go callDealy() + go proxyDisable() + err = <-proxyErrChan + Require(t, err) + err = <-errChan + if err != nil { + Fail(t, "call during proxyDisable failed with retries:", err) + } + + config.Retries = 0 + go callDealy() + go proxyReset() + err = <-proxyErrChan + Require(t, err) + err = <-errChan + if err == nil { + Fail(t, "call during proxyReset succeeded without retries") + } + + config.Retries = 3 + go callDealy() + go proxyReset() + err = <-proxyErrChan + Require(t, err) + err = <-errChan + if err != nil { + Fail(t, "call during proxyReset failed with retries:", err) + } +}