VRF-664:add Slack notification for VRF v2 Plus WASP test (#11063)

* VRF-664:add Slack notification for VRF v2 Plus WASP test * VRF-664:fixing import cycle * VRF-664:cleanup * VRF-664:adding slack channel secret
smartcontractkit · Oct 25, 2023 · d31223c · d31223c
1 parent 93a4ef9
commit d31223c
Show file tree

Hide file tree

Showing 7 changed files with 228 additions and 58 deletions.
diff --git a/...rkflows/on-demand-vrfv2plus-load-test.yml → .../on-demand-vrfv2plus-performance-test.yml b/...rkflows/on-demand-vrfv2plus-load-test.yml → .../on-demand-vrfv2plus-performance-test.yml
@@ -1,4 +1,4 @@
-name: On Demand VRFV2 Plus Load Test
+name: On Demand VRFV2 Plus Performance Test
 on:
   workflow_dispatch:
     inputs:
@@ -49,8 +49,8 @@ on:
           - "Spike"
       testDuration:
         description: Duration of the test (time string)
-        required: false
-        default: 1m
+        required: true
+        default: 5m
       useExistingEnv:
         description: Set `true` to use existing environment or `false` to deploy CL node and all contracts
         required: false
@@ -59,8 +59,8 @@ on:
         description: TOML config in base64 (Needed when overriding config or providing contract addresses for existing env)
         required: false
 jobs:
-  vrfv2plus_load_test:
-    name: ${{ inputs.network }} VRFV2 Plus Load Test
+  vrfv2plus_performance_test:
+    name: ${{ inputs.network }} VRFV2 Plus Performance Test
     environment: integration
     runs-on: ubuntu20.04-8cores-32GB
     permissions:
@@ -80,6 +80,8 @@ jobs:
       REF_NAME: ${{ github.head_ref || github.ref_name }}
       CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }}
       CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
+      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}
       WASP_LOG_LEVEL: info
     steps:
       - name: Collect Metrics
@@ -88,8 +90,15 @@ jobs:
         with:
           basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
           hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
-          this-job-name: ${{ inputs.network }} VRFV2 Plus Load Test
+          this-job-name: ${{ inputs.network }} VRFV2 Plus Performance Test
         continue-on-error: true
+      - name: Setup Push Tag
+        shell: bash
+        run: |
+          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
+          echo "\`${{ inputs.chainlinkVersion }}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
+          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
       - name: Get Inputs
         run: |
           EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH)
@@ -111,7 +120,7 @@ jobs:
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         with:
-          test_command_to_run: cd ./integration-tests && go test -v -count=1 -timeout 6h -run TestVRFV2PlusLoad/vrfv2plus_soak_test ./load/vrfv2plus
+          test_command_to_run: cd ./integration-tests && go test -v -count=1 -timeout 6h -run TestVRFV2PlusPerformance/vrfv2plus_performance_test ./load/vrfv2plus
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ inputs.chainlinkImage }}
           cl_image_tag: ${{ inputs.chainlinkVersion }}

diff --git a/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go b/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go
@@ -18,6 +18,8 @@ type VRFV2PlusConfig struct {
 	FulfillmentFlatFeeLinkPPM       uint32  `envconfig:"FULFILLMENT_FLAT_FEE_LINK_PPM" default:"500"`             // Flat fee in ppm for LINK for the VRF Coordinator config
 	FulfillmentFlatFeeNativePPM     uint32  `envconfig:"FULFILLMENT_FLAT_FEE_NATIVE_PPM" default:"500"`           // Flat fee in ppm for native currency for the VRF Coordinator config
 
+	NumberOfSubToCreate int `envconfig:"NUMBER_OF_SUB_TO_CREATE" default:"1"` // Number of subscriptions to create
+
 	RandomnessRequestCountPerRequest          uint16 `envconfig:"RANDOMNESS_REQUEST_COUNT_PER_REQUEST" default:"1"`           // How many randomness requests to send per request
 	RandomnessRequestCountPerRequestDeviation uint16 `envconfig:"RANDOMNESS_REQUEST_COUNT_PER_REQUEST_DEVIATION" default:"0"` // How many randomness requests to send per request
 

diff --git a/integration-tests/load/vrfv2plus/config.go b/integration-tests/load/vrfv2plus/config.go
@@ -12,6 +12,10 @@ import (
 
 const (
 	DefaultConfigFilename = "config.toml"
+	SoakTestType          = "Soak"
+	LoadTestType          = "Load"
+	StressTestType        = "Stress"
+	SpikeTestType         = "Spike"
 
 	ErrReadPerfConfig                    = "failed to read TOML config for performance tests"
 	ErrUnmarshalPerfConfig               = "failed to unmarshal TOML config for performance tests"
@@ -38,7 +42,6 @@ type ExistingEnvConfig struct {
 
 type NewEnvConfig struct {
 	Funding
-	NumberOfSubToCreate int `toml:"number_of_sub_to_create"`
 }
 
 type Common struct {
@@ -68,6 +71,8 @@ type Spike struct {
 }
 
 type PerformanceTestConfig struct {
+	NumberOfSubToCreate int `toml:"number_of_sub_to_create"`
+
 	RPS int64 `toml:"rps"`
 	//Duration *models.Duration `toml:"duration"`
 	RateLimitUnitDuration                     *models.Duration `toml:"rate_limit_unit_duration"`
@@ -101,24 +106,28 @@ func ReadConfig() (*PerformanceConfig, error) {
 	return cfg, nil
 }
 
-func SetPerformanceTestConfig(vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, cfg *PerformanceConfig) {
-	switch os.Getenv("TEST_TYPE") {
-	case "Soak":
+func SetPerformanceTestConfig(testType string, vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, cfg *PerformanceConfig) {
+	switch testType {
+	case SoakTestType:
+		vrfv2PlusConfig.NumberOfSubToCreate = cfg.Soak.NumberOfSubToCreate
 		vrfv2PlusConfig.RPS = cfg.Soak.RPS
 		vrfv2PlusConfig.RateLimitUnitDuration = cfg.Soak.RateLimitUnitDuration.Duration()
 		vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Soak.RandomnessRequestCountPerRequest
 		vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation = cfg.Soak.RandomnessRequestCountPerRequestDeviation
-	case "Load":
+	case LoadTestType:
+		vrfv2PlusConfig.NumberOfSubToCreate = cfg.Load.NumberOfSubToCreate
 		vrfv2PlusConfig.RPS = cfg.Load.RPS
 		vrfv2PlusConfig.RateLimitUnitDuration = cfg.Load.RateLimitUnitDuration.Duration()
 		vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Load.RandomnessRequestCountPerRequest
 		vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation = cfg.Load.RandomnessRequestCountPerRequestDeviation
-	case "Stress":
+	case StressTestType:
+		vrfv2PlusConfig.NumberOfSubToCreate = cfg.Stress.NumberOfSubToCreate
 		vrfv2PlusConfig.RPS = cfg.Stress.RPS
 		vrfv2PlusConfig.RateLimitUnitDuration = cfg.Stress.RateLimitUnitDuration.Duration()
 		vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Stress.RandomnessRequestCountPerRequest
 		vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation = cfg.Stress.RandomnessRequestCountPerRequestDeviation
-	case "Spike":
+	case SpikeTestType:
+		vrfv2PlusConfig.NumberOfSubToCreate = cfg.Spike.NumberOfSubToCreate
 		vrfv2PlusConfig.RPS = cfg.Spike.RPS
 		vrfv2PlusConfig.RateLimitUnitDuration = cfg.Spike.RateLimitUnitDuration.Duration()
 		vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Spike.RandomnessRequestCountPerRequest

diff --git a/integration-tests/load/vrfv2plus/config.toml b/integration-tests/load/vrfv2plus/config.toml
@@ -6,7 +6,7 @@ minimum_confirmations = 3
 sub_funds_link = 1000
 sub_funds_native = 1000
 node_funds = 10
-number_of_sub_to_create = 10
+
 
 [ExistingEnvConfig]
 coordinator_address = "0x4931Ce2e341398c8eD8A5D0F6ADb920476D6DaBb"
@@ -21,24 +21,28 @@ rate_limit_unit_duration = "6s"
 rps = 1
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 1
 
 # approx 60 RPM - 1 tx request with 4 rand requests in each tx every 3 seconds
 [Load]
 rate_limit_unit_duration = "3s"
 rps = 1
 randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 10
 
 # approx 540 RPM - 3 tx requests per second with 4 rand requests in each tx
 [Stress]
-rate_limit_unit_duration = "0"
+rate_limit_unit_duration = "1s"
 rps = 3
 randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 20
 
 # approx 150 RPM - 1 tx request with 150 rand requests in each tx every 60 seconds
 [Spike]
 rate_limit_unit_duration = "1m"
 rps = 1
 randomness_request_count_per_request = 150 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 1
diff --git a/integration-tests/load/vrfv2plus/onchain_monitoring.go b/integration-tests/load/vrfv2plus/onchain_monitoring.go
@@ -3,7 +3,7 @@ package loadvrfv2plus
 import (
 	"context"
 	"github.com/rs/zerolog/log"
-	"github.com/smartcontractkit/chainlink/integration-tests/actions/vrfv2plus"
+	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/wasp"
 	"testing"
 	"time"
@@ -18,11 +18,12 @@ const (
 	ErrLokiPush   = "failed to push monitoring metrics to Loki"
 )
 
-func MonitorLoadStats(lc *wasp.LokiClient, vrfv2PlusContracts *vrfv2plus.VRFV2_5Contracts, labels map[string]string) {
+func MonitorLoadStats(lc *wasp.LokiClient, consumer contracts.VRFv2PlusLoadTestConsumer, labels map[string]string) {
 	go func() {
 		for {
 			time.Sleep(1 * time.Second)
-			SendLoadTestMetricsToLoki(vrfv2PlusContracts, lc, labels)
+			metrics := GetLoadTestMetrics(consumer)
+			SendMetricsToLoki(metrics, lc, labels)
 		}
 	}()
 }
@@ -38,13 +39,16 @@ func UpdateLabels(labels map[string]string, t *testing.T) map[string]string {
 	return updatedLabels
 }
 
-func SendLoadTestMetricsToLoki(vrfv2PlusContracts *vrfv2plus.VRFV2_5Contracts, lc *wasp.LokiClient, updatedLabels map[string]string) {
-	//todo - should work with multiple consumers and consumers having different keyhashes and wallets
-	metrics, err := vrfv2PlusContracts.LoadTestConsumers[0].GetLoadTestMetrics(context.Background())
-	if err != nil {
-		log.Error().Err(err).Msg(ErrMetrics)
-	}
+func SendMetricsToLoki(metrics *contracts.VRFLoadTestMetrics, lc *wasp.LokiClient, updatedLabels map[string]string) {
 	if err := lc.HandleStruct(wasp.LabelsMapToModel(updatedLabels), time.Now(), metrics); err != nil {
 		log.Error().Err(err).Msg(ErrLokiPush)
 	}
 }
+
+func GetLoadTestMetrics(consumer contracts.VRFv2PlusLoadTestConsumer) *contracts.VRFLoadTestMetrics {
+	metrics, err := consumer.GetLoadTestMetrics(context.Background())
+	if err != nil {
+		log.Error().Err(err).Msg(ErrMetrics)
+	}
+	return metrics
+}