From 31c0cb7c474dedcc31a340db58d27668f7d5141a Mon Sep 17 00:00:00 2001 From: Lukasz <120112546+lukaszcl@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:03:27 +0200 Subject: [PATCH] Add new OCR tests with RPC node failures (#13624) * Add ocr soak test with rpc node down for some period * Remove local replace from go mod tidy * Cleanup * Add Grafana annotation if configured * Refactor * Do not show error when chaos already deleted * Bump ctf * Update chainlink_node_funding * Add tests to makefile * Fix lint * Fail test immediately if chaos cannot be created * Fix * Make forwarderFlow an opt * Fix lint --- integration-tests/Makefile | 10 ++ integration-tests/go.mod | 13 +- integration-tests/go.sum | 22 ++- integration-tests/load/go.mod | 6 +- integration-tests/load/go.sum | 20 ++- integration-tests/soak/forwarder_ocr_test.go | 2 +- integration-tests/soak/ocr_test.go | 170 +++++++++++++++++-- integration-tests/testconfig/ocr/ocr.toml | 32 +++- integration-tests/testsetups/ocr.go | 112 +++++++++++- 9 files changed, 328 insertions(+), 59 deletions(-) diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 5b3b3a2aae2..9a85d52125c 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -168,6 +168,16 @@ test_soak_ocr_gas_limit_change: . ./scripts/check_base64_env_var.sh go test -v -count=1 -run ^TestOCRSoak_ChangeBlockGasLimit$$ ./soak +.PHONY: test_soak_ocr_rpc_down_all_cl_nodes +test_soak_ocr_rpc_down_all_cl_nodes: + . ./scripts/check_base64_env_var.sh + go test -v -count=1 -run ^TestOCRSoak_RPCDownForAllCLNodes$$ ./soak + +.PHONY: test_soak_ocr_rpc_down_half_cl_nodes +test_soak_ocr_rpc_down_half_cl_nodes: + . ./scripts/check_base64_env_var.sh + go test -v -count=1 -run ^TestOCRSoak_RPCDownForHalfCLNodes$$ ./soak + .PHONY: test_soak_forwarder_ocr test_soak_forwarder_ocr: . ./scripts/check_base64_env_var.sh diff --git a/integration-tests/go.mod b/integration-tests/go.mod index c622c6bf9d3..4924b08ad59 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -8,6 +8,7 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ require ( github.com/avast/retry-go/v4 v4.5.1 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df + github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 github.com/cli/go-gh/v2 v2.0.0 github.com/ethereum/go-ethereum v1.13.8 github.com/fxamacker/cbor/v2 v2.5.0 @@ -21,16 +22,17 @@ require ( github.com/onsi/gomega v1.30.0 github.com/pelletier/go-toml/v2 v2.2.2 github.com/pkg/errors v0.9.1 - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/scylladb/go-reflectx v1.0.1 github.com/segmentio/ksuid v1.0.4 github.com/shopspring/decimal v1.3.1 github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240618210005-a88f179ffc16 - github.com/smartcontractkit/chainlink-testing-framework v1.30.9 + github.com/smartcontractkit/chainlink-testing-framework v1.31.1 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 + github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c github.com/smartcontractkit/seth v1.0.12 github.com/smartcontractkit/wasp v0.4.7 @@ -44,6 +46,7 @@ require ( golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 gopkg.in/guregu/null.v4 v4.0.0 + k8s.io/apimachinery v0.28.2 ) // avoids ambigious imports of indirect dependencies @@ -110,7 +113,6 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/chzyer/readline v1.5.1 // indirect @@ -205,6 +207,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.15.5 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect + github.com/go-test/deep v1.0.8 // indirect github.com/go-webauthn/webauthn v0.9.4 // indirect github.com/go-webauthn/x v0.1.5 // indirect github.com/goccy/go-json v0.10.2 // indirect @@ -380,7 +383,7 @@ require ( github.com/smartcontractkit/chainlink-feeds v0.0.0-20240522213638-159fb2d99917 // indirect github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 // indirect - github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 // indirect + github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 // indirect github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 // indirect github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1 // indirect github.com/smartcontractkit/wsrpc v0.8.1 // indirect @@ -413,6 +416,7 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/umbracle/fastrlp v0.0.0-20220527094140-59d5dd30e722 // indirect + github.com/urfave/cli/v2 v2.27.1 // indirect github.com/valyala/fastjson v1.4.1 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -469,7 +473,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.28.2 // indirect k8s.io/apiextensions-apiserver v0.25.3 // indirect - k8s.io/apimachinery v0.28.2 // indirect k8s.io/cli-runtime v0.25.11 // indirect k8s.io/client-go v0.28.2 // indirect k8s.io/component-base v0.26.2 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 8a67373725e..8fe1e02bf36 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -652,8 +652,9 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-webauthn/webauthn v0.9.4 h1:YxvHSqgUyc5AK2pZbqkWWR55qKeDPhP8zLDr6lpIc2g= github.com/go-webauthn/webauthn v0.9.4/go.mod h1:LqupCtzSef38FcxzaklmOn7AykGKhAhr9xlRbdbgnTw= github.com/go-webauthn/x v0.1.5 h1:V2TCzDU2TGLd0kSZOXdrqDVV5JB9ILnKxA9S53CSBw0= @@ -1196,6 +1197,7 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1455,8 +1457,8 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= @@ -1524,16 +1526,18 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.9 h1:U8TqnPn2pfe+jSojeZL58sXBgfHFBHRsnm3izLthaYw= -github.com/smartcontractkit/chainlink-testing-framework v1.30.9/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= +github.com/smartcontractkit/chainlink-testing-framework v1.31.1 h1:Qqo5VngCqbHQPfQKZneAN0L1dWXOWWd074Oo9Bex/Q0= +github.com/smartcontractkit/chainlink-testing-framework v1.31.1/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0= +github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 h1:ZEhn2Yo1jY4hqy8nasDL4k4pNtopT3rS3Ap1GDb7ODc= +github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37/go.mod h1:/kFr0D7SI/vueXl1N03uzOun4nViGPFRyA5X6eL3jXw= github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c h1:lIyMbTaF2H0Q71vkwZHX/Ew4KF2BxiKhqEXwF8rn+KI= github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= github.com/smartcontractkit/seth v1.0.12 h1:iVdgMx42XWanPPnBaM5StR4c1XsTr/0/B/kKRZL5BsY= @@ -1669,8 +1673,8 @@ github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFs github.com/unrolled/secure v1.13.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= +github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 1d89efcb01c..c9c286b941b 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -13,11 +13,11 @@ require ( github.com/go-resty/resty/v2 v2.11.0 github.com/pelletier/go-toml/v2 v2.2.2 github.com/pkg/errors v0.9.1 - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240618210005-a88f179ffc16 - github.com/smartcontractkit/chainlink-testing-framework v1.30.9 + github.com/smartcontractkit/chainlink-testing-framework v1.31.1 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c @@ -369,7 +369,7 @@ require ( github.com/smartcontractkit/chainlink-feeds v0.0.0-20240522213638-159fb2d99917 // indirect github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 // indirect - github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 // indirect + github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 // indirect github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 // indirect github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 // indirect github.com/smartcontractkit/wsrpc v0.8.1 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 1b339816dea..df7496796c5 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -646,8 +646,9 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-webauthn/webauthn v0.9.4 h1:YxvHSqgUyc5AK2pZbqkWWR55qKeDPhP8zLDr6lpIc2g= github.com/go-webauthn/webauthn v0.9.4/go.mod h1:LqupCtzSef38FcxzaklmOn7AykGKhAhr9xlRbdbgnTw= github.com/go-webauthn/x v0.1.5 h1:V2TCzDU2TGLd0kSZOXdrqDVV5JB9ILnKxA9S53CSBw0= @@ -1190,6 +1191,7 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1445,8 +1447,8 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= @@ -1514,10 +1516,10 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.9 h1:U8TqnPn2pfe+jSojeZL58sXBgfHFBHRsnm3izLthaYw= -github.com/smartcontractkit/chainlink-testing-framework v1.30.9/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= +github.com/smartcontractkit/chainlink-testing-framework v1.31.1 h1:Qqo5VngCqbHQPfQKZneAN0L1dWXOWWd074Oo9Bex/Q0= +github.com/smartcontractkit/chainlink-testing-framework v1.31.1/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs= github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306 h1:ko88+ZznniNJZbZPWAvHQU8SwKAdHngdDZ+pvVgB5ss= @@ -1657,8 +1659,8 @@ github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFs github.com/unrolled/secure v1.13.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= +github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= diff --git a/integration-tests/soak/forwarder_ocr_test.go b/integration-tests/soak/forwarder_ocr_test.go index b5355a6c3f5..37837c666aa 100644 --- a/integration-tests/soak/forwarder_ocr_test.go +++ b/integration-tests/soak/forwarder_ocr_test.go @@ -25,7 +25,7 @@ ForwardersEnabled = true` config, err := tc.GetConfig("Soak", tc.OCR) require.NoError(t, err, "Error getting config") - ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, true) + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, testsetups.WithForwarderFlow(true)) require.NoError(t, err, "Error creating soak test") ocrSoakTest.DeployEnvironment(customNetworkTOML, &config) if ocrSoakTest.Environment().WillUseRemoteRunner() { diff --git a/integration-tests/soak/ocr_test.go b/integration-tests/soak/ocr_test.go index f0571fc63a1..290cc00255c 100644 --- a/integration-tests/soak/ocr_test.go +++ b/integration-tests/soak/ocr_test.go @@ -1,12 +1,21 @@ package soak import ( + "fmt" "testing" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-testing-framework/logging" + "time" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/google/uuid" + "github.com/smartcontractkit/havoc/k8schaos" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" "github.com/smartcontractkit/chainlink/integration-tests/actions" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" "github.com/smartcontractkit/chainlink/integration-tests/testsetups" @@ -22,58 +31,183 @@ func TestOCRSoak(t *testing.T) { // fmt.Println("---------------------") config, err := tc.GetConfig("Soak", tc.OCR) require.NoError(t, err, "Error getting config") - runOCRSoakTest(t, config, customNetworkTOML) + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config) + require.NoError(t, err, "Error creating OCR soak test") + runOCRSoakTest(t, ocrSoakTest, config, customNetworkTOML) } func TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled(t *testing.T) { config, err := tc.GetConfig(t.Name(), tc.OCR) require.NoError(t, err, "Error getting config") - runOCRSoakTest(t, config, "") + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config) + require.NoError(t, err, "Error creating OCR soak test") + runOCRSoakTest(t, ocrSoakTest, config, "") } func TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled(t *testing.T) { config, err := tc.GetConfig(t.Name(), tc.OCR) require.NoError(t, err, "Error getting config") - runOCRSoakTest(t, config, "") + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config) + require.NoError(t, err, "Error creating OCR soak test") + runOCRSoakTest(t, ocrSoakTest, config, "") } func TestOCRSoak_GasSpike(t *testing.T) { config, err := tc.GetConfig(t.Name(), tc.OCR) require.NoError(t, err, "Error getting config") - runOCRSoakTest(t, config, "") + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config) + require.NoError(t, err, "Error creating OCR soak test") + runOCRSoakTest(t, ocrSoakTest, config, "") } // TestOCRSoak_ChangeBlockGasLimit changes next block gas limit and sets it to percentage of last gasUsed in previous block creating congestion func TestOCRSoak_ChangeBlockGasLimit(t *testing.T) { config, err := tc.GetConfig(t.Name(), tc.OCR) require.NoError(t, err, "Error getting config") - runOCRSoakTest(t, config, "") + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config) + require.NoError(t, err, "Error creating OCR soak test") + runOCRSoakTest(t, ocrSoakTest, config, "") +} + +// TestOCRSoak_RPCDownForAllCLNodes simulates a network chaos by bringing down network to RPC node for all Chainlink Nodes +func TestOCRSoak_RPCDownForAllCLNodes(t *testing.T) { + config, err := tc.GetConfig(t.Name(), tc.OCR) + require.NoError(t, err, "Error getting config") + require.True(t, config.Network.IsSimulatedGethSelected(), "This test requires simulated geth") + + namespace := fmt.Sprintf("%s-%s", "soak-ocr-simulated-geth", uuid.NewString()[0:5]) + chaos, err := gethNetworkDownChaos(GethNetworkDownChaosOpts{ + DelayCreate: time.Minute * 2, + Duration: time.Minute * 5, + Name: "ocr-soak-test-geth-network-down-all-nodes", + Description: "Geth Network Down For All Chainlink Nodes", + Namespace: namespace, + TargetSelector: v1alpha1.PodSelector{ + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{namespace}, + LabelSelectors: map[string]string{"app": "chainlink-0"}, + }, + }, + Mode: v1alpha1.AllMode, + }, + }) + require.NoError(t, err, "Error creating chaos") + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, + testsetups.WithNamespace(namespace), + testsetups.WithChaos([]*k8schaos.Chaos{chaos}), + ) + require.NoError(t, err, "Error creating OCR soak test") + runOCRSoakTest(t, ocrSoakTest, config, "") +} + +// TestOCRSoak_RPCDownForAllCLNodes simulates a network chaos by bringing down network to RPC node for 50% of Chainlink Nodes +func TestOCRSoak_RPCDownForHalfCLNodes(t *testing.T) { + config, err := tc.GetConfig(t.Name(), tc.OCR) + require.NoError(t, err, "Error getting config") + require.True(t, config.Network.IsSimulatedGethSelected(), "This test requires simulated geth") + + namespace := fmt.Sprintf("%s-%s", "soak-ocr-simulated-geth", uuid.NewString()[0:5]) + chaos, err := gethNetworkDownChaos(GethNetworkDownChaosOpts{ + DelayCreate: time.Minute * 2, + Duration: time.Minute * 5, + Name: "ocr-soak-test-geth-network-down-half-nodes", + Description: "Geth Network Down For 50 Percent Of Chainlink Nodes", + Namespace: namespace, + TargetSelector: v1alpha1.PodSelector{ + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{namespace}, + LabelSelectors: map[string]string{"app": "chainlink-0"}, + }, + }, + Mode: v1alpha1.FixedPercentMode, + Value: "50", + }, + }) + require.NoError(t, err, "Error creating chaos") + ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, + testsetups.WithNamespace(namespace), + testsetups.WithChaos([]*k8schaos.Chaos{chaos}), + ) + require.NoError(t, err, "Error creating OCR soak test") + runOCRSoakTest(t, ocrSoakTest, config, "") } -func runOCRSoakTest(t *testing.T, config tc.TestConfig, customNetworkTOML string) { +func runOCRSoakTest(t *testing.T, test *testsetups.OCRSoakTest, config tc.TestConfig, customNetworkTOML string) { l := logging.GetTestLogger(t) l.Info().Str("test", t.Name()).Msg("Starting OCR soak test") - - ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false) - require.NoError(t, err, "Error creating soak test") - if !ocrSoakTest.Interrupted() { - ocrSoakTest.DeployEnvironment(customNetworkTOML, &config) + if !test.Interrupted() { + test.DeployEnvironment(customNetworkTOML, &config) } - if ocrSoakTest.Environment().WillUseRemoteRunner() { + if test.Environment().WillUseRemoteRunner() { return } t.Cleanup(func() { - if err := actions.TeardownRemoteSuite(ocrSoakTest.TearDownVals(t)); err != nil { + if err := actions.TeardownRemoteSuite(test.TearDownVals(t)); err != nil { l.Error().Err(err).Msg("Error tearing down environment") } }) - if ocrSoakTest.Interrupted() { - err = ocrSoakTest.LoadState() + if test.Interrupted() { + err := test.LoadState() require.NoError(t, err, "Error loading state") - ocrSoakTest.Resume() + test.Resume() } else { - ocrSoakTest.Setup(&config) - ocrSoakTest.Run() + test.Setup(&config) + test.Run() } } + +type GethNetworkDownChaosOpts struct { + Name string + Namespace string + Description string + TargetSelector v1alpha1.PodSelector + DelayCreate time.Duration + Duration time.Duration +} + +func gethNetworkDownChaos(opts GethNetworkDownChaosOpts) (*k8schaos.Chaos, error) { + k8sClient, err := k8schaos.NewChaosMeshClient() + if err != nil { + return nil, err + } + return k8schaos.NewChaos(k8schaos.ChaosOpts{ + Description: opts.Description, + DelayCreate: opts.DelayCreate, + Object: &v1alpha1.NetworkChaos{ + TypeMeta: metav1.TypeMeta{ + Kind: "NetworkChaos", + APIVersion: "chaos-mesh.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opts.Name, + Namespace: opts.Namespace, + }, + Spec: v1alpha1.NetworkChaosSpec{ + Action: v1alpha1.LossAction, + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{opts.Namespace}, + LabelSelectors: map[string]string{"app": "geth"}, + }, + }, + }, + Duration: ptr.Ptr(opts.Duration.String()), + Direction: v1alpha1.Both, + Target: &opts.TargetSelector, + TcParameter: v1alpha1.TcParameter{ + Loss: &v1alpha1.LossSpec{ + Loss: "100", + }, + }, + }, + }, + Client: k8sClient, + Logger: &k8schaos.Logger, + }) + +} diff --git a/integration-tests/testconfig/ocr/ocr.toml b/integration-tests/testconfig/ocr/ocr.toml index 6c5745807b4..4e280e88f04 100644 --- a/integration-tests/testconfig/ocr/ocr.toml +++ b/integration-tests/testconfig/ocr/ocr.toml @@ -100,7 +100,7 @@ enabled = true depth = 15 delay_create = "3s" [TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled.Common] -chainlink_node_funding = 100 +chainlink_node_funding = 0.5 [TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled.OCR] [TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled.OCR.Common] test_duration="15m" @@ -128,7 +128,7 @@ enabled = true depth = 15 delay_create = "3s" [TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled.Common] -chainlink_node_funding = 100 +chainlink_node_funding = 0.5 [TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled.OCR] [TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled.OCR.Common] test_duration="15m" @@ -139,7 +139,7 @@ time_between_rounds="1m" # OCR soak test configuration with gas spike on Anvil network [TestOCRSoak_GasSpike.Common] -chainlink_node_funding = 100 +chainlink_node_funding = 0.5 [TestOCRSoak_GasSpike.OCR.Common] test_duration="15m" [TestOCRSoak_GasSpike.OCR.Soak] @@ -158,7 +158,7 @@ duration = "3m" # OCR soak test configuration with change to gas limit on Anvil network [TestOCRSoak_ChangeBlockGasLimit.Common] -chainlink_node_funding = 100 +chainlink_node_funding = 0.5 [TestOCRSoak_ChangeBlockGasLimit.OCR.Common] test_duration="15m" [TestOCRSoak_ChangeBlockGasLimit.OCR.Soak] @@ -171,4 +171,26 @@ selected_networks=["Anvil"] enabled = true next_gas_limit_percentage = 0.5 delay_create = "1m" -duration = "3m" \ No newline at end of file +duration = "3m" + +[TestOCRSoak_RPCDownForAllCLNodes.Common] +chainlink_node_funding = 0.5 +[TestOCRSoak_RPCDownForAllCLNodes.OCR.Common] +test_duration="15m" +[TestOCRSoak_RPCDownForAllCLNodes.OCR.Soak] +ocr_version="1" +number_of_contracts=2 +time_between_rounds="1m" +[TestOCRSoak_RPCDownForAllCLNodes.Network] +selected_networks=["simulated"] + +[TestOCRSoak_RPCDownForHalfCLNodes.Common] +chainlink_node_funding = 0.5 +[TestOCRSoak_RPCDownForHalfCLNodes.OCR.Common] +test_duration="15m" +[TestOCRSoak_RPCDownForHalfCLNodes.OCR.Soak] +ocr_version="1" +number_of_contracts=2 +time_between_rounds="1m" +[TestOCRSoak_RPCDownForHalfCLNodes.Network] +selected_networks=["simulated"] \ No newline at end of file diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index bcad989e171..be11de6313b 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -20,6 +20,7 @@ import ( geth "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/google/uuid" "github.com/pelletier/go-toml/v2" "github.com/rs/zerolog" "github.com/smartcontractkit/seth" @@ -28,6 +29,8 @@ import ( "github.com/smartcontractkit/libocr/gethwrappers/offchainaggregator" "github.com/smartcontractkit/libocr/gethwrappers2/ocr2aggregator" + "github.com/smartcontractkit/havoc/k8schaos" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" ctf_client "github.com/smartcontractkit/chainlink-testing-framework/client" ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" @@ -88,13 +91,33 @@ type OCRSoakTest struct { reorgHappened bool // flag to indicate if a reorg happened during the test gasSpikeSimulationHappened bool // flag to indicate if a gas spike simulation happened during the test gasLimitSimulationHappened bool // flag to indicate if a gas limit simulation happened during the test + chaosList []*k8schaos.Chaos // list of chaos simulations to run during the test +} + +type OCRSoakTestOption = func(c *OCRSoakTest) + +func WithChaos(chaosList []*k8schaos.Chaos) OCRSoakTestOption { + return func(c *OCRSoakTest) { + c.chaosList = chaosList + } +} + +func WithNamespace(ns string) OCRSoakTestOption { + return func(c *OCRSoakTest) { + c.namespace = ns + } +} + +func WithForwarderFlow(forwarderFlow bool) OCRSoakTestOption { + return func(c *OCRSoakTest) { + c.OperatorForwarderFlow = forwarderFlow + } } // NewOCRSoakTest creates a new OCR soak test to setup and run -func NewOCRSoakTest(t *testing.T, config *tc.TestConfig, forwarderFlow bool) (*OCRSoakTest, error) { +func NewOCRSoakTest(t *testing.T, config *tc.TestConfig, opts ...OCRSoakTestOption) (*OCRSoakTest, error) { test := &OCRSoakTest{ - Config: config, - OperatorForwarderFlow: forwarderFlow, + Config: config, TestReporter: testreporters.OCRSoakTestReporter{ OCRVersion: *config.OCR.Soak.OCRVersion, StartTime: time.Now(), @@ -107,22 +130,35 @@ func NewOCRSoakTest(t *testing.T, config *tc.TestConfig, forwarderFlow bool) (*O ocrV1InstanceMap: make(map[string]contracts.OffchainAggregator), ocrV2InstanceMap: make(map[string]contracts.OffchainAggregatorV2), } + for _, opt := range opts { + opt(test) + } + t.Cleanup(func() { + test.deleteChaosSimulations() + }) return test, test.ensureInputValues() } // DeployEnvironment deploys the test environment, starting all Chainlink nodes and other components for the test func (o *OCRSoakTest) DeployEnvironment(customChainlinkNetworkTOML string, ocrTestConfig tt.OcrTestConfig) { nodeNetwork := networks.MustGetSelectedNetworkConfig(ocrTestConfig.GetNetworkConfig())[0] // Environment currently being used to soak test on - nsPre := fmt.Sprintf("soak-ocr-v%s-", *ocrTestConfig.GetOCRConfig().Soak.OCRVersion) - if o.OperatorForwarderFlow { - nsPre = fmt.Sprintf("%sforwarder-", nsPre) + + // Define namespace if default not set + if o.namespace == "" { + nsPre := fmt.Sprintf("soak-ocr-v%s-", *ocrTestConfig.GetOCRConfig().Soak.OCRVersion) + if o.OperatorForwarderFlow { + nsPre = fmt.Sprintf("%sforwarder-", nsPre) + } + + nsPre = fmt.Sprintf("%s%s", nsPre, strings.ReplaceAll(strings.ToLower(nodeNetwork.Name), " ", "-")) + nsPre = strings.ReplaceAll(nsPre, "_", "-") + + o.namespace = fmt.Sprintf("%s-%s", nsPre, uuid.NewString()[0:5]) } - nsPre = fmt.Sprintf("%s%s", nsPre, strings.ReplaceAll(strings.ToLower(nodeNetwork.Name), " ", "-")) - nsPre = strings.ReplaceAll(nsPre, "_", "-") baseEnvironmentConfig := &environment.Config{ TTL: time.Hour * 720, // 30 days, - NamespacePrefix: nsPre, + Namespace: o.namespace, Test: o.t, PreventPodEviction: true, } @@ -645,6 +681,21 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) { } } + // Schedule chaos simulations if needed + if len(o.chaosList) > 0 { + for _, chaos := range o.chaosList { + chaos.Create(context.Background()) + chaos.AddListener(k8schaos.NewChaosLogger(o.log)) + chaos.AddListener(ocrTestChaosListener{t: o.t}) + // Add Grafana annotation if configured + if o.Config.Logging.Grafana != nil && o.Config.Logging.Grafana.BaseUrl != nil && o.Config.Logging.Grafana.BearerToken != nil && o.Config.Logging.Grafana.DashboardUID != nil { + chaos.AddListener(k8schaos.NewSingleLineGrafanaAnnotator( + *o.Config.Logging.Grafana.BaseUrl, *o.Config.Logging.Grafana.BearerToken, + *o.Config.Logging.Grafana.DashboardUID, o.log)) + } + } + } + for { select { case <-interruption: @@ -658,6 +709,7 @@ func (o *OCRSoakTest) testLoop(testDuration time.Duration, newValue int) { o.log.Error().Err(err).Msg("Error saving state") } o.log.Warn().Str("Time Taken", time.Since(saveStart).String()).Msg("Saved state") + o.deleteChaosSimulations() os.Exit(interruptedExitCode) // Exit with interrupted code to indicate test was interrupted, not just a normal failure case <-endTest: return @@ -741,6 +793,18 @@ func (o *OCRSoakTest) startAnvilGasLimitSimulation(network blockchain.EVMNetwork o.gasLimitSimulationHappened = true } +// Delete k8s chaos objects it any of them still exist +// This is needed to clean up the chaos objects if the test is interrupted or it finishes +func (o *OCRSoakTest) deleteChaosSimulations() { + for _, chaos := range o.chaosList { + err := chaos.Delete(context.Background()) + // Check if the error is because the chaos object is already deleted + if err != nil && !strings.Contains(err.Error(), "not found") { + o.log.Error().Err(err).Msg("Error deleting chaos object") + } + } +} + // setFilterQuery to look for all events that happened func (o *OCRSoakTest) setFilterQuery() { ocrAddresses := o.getContractAddresses() @@ -1009,3 +1073,33 @@ func (o *OCRSoakTest) getContractAddresses() []common.Address { return contractAddresses } + +type ocrTestChaosListener struct { + t *testing.T +} + +func (l ocrTestChaosListener) OnChaosCreated(_ k8schaos.Chaos) { +} + +func (l ocrTestChaosListener) OnChaosCreationFailed(chaos k8schaos.Chaos, reason error) { + // Fail the test if chaos creation fails during chaos simulation + require.FailNow(l.t, "Error creating chaos simulation", reason.Error(), chaos) +} + +func (l ocrTestChaosListener) OnChaosStarted(_ k8schaos.Chaos) { +} + +func (l ocrTestChaosListener) OnChaosPaused(_ k8schaos.Chaos) { +} + +func (l ocrTestChaosListener) OnChaosEnded(_ k8schaos.Chaos) { +} + +func (l ocrTestChaosListener) OnChaosStatusUnknown(_ k8schaos.Chaos) { +} + +func (l ocrTestChaosListener) OnScheduleCreated(_ k8schaos.Schedule) { +} + +func (l ocrTestChaosListener) OnScheduleDeleted(_ k8schaos.Schedule) { +}