webrtc-load-test: Create CI action to run a load test (#339)

* Create CI action to run a load test * Force CI to run on push * Remove push trigger and discord logs * Add some dirty debug logging * Use custom container version * Try toJSON to escape the JSON envs * Some saner defaults for inputs * Back to image * Revert "Back to image" This reverts commit ece4246. * Revert "Try toJSON to escape the JSON envs" This reverts commit 5c60fec. * Ignore commas on env vars! * Allow running the test in prod or staging * Use single equals * Uncomment the discord notifications
livepeer · Nov 15, 2023 · cd57099 · cd57099
1 parent eab54d1
commit cd57099
Show file tree

Hide file tree

Showing 5 changed files with 214 additions and 4 deletions.
diff --git a/.github/workflows/webrtc-load-test.yaml b/.github/workflows/webrtc-load-test.yaml
@@ -0,0 +1,208 @@
+name: Run a WebRTC Load Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      production:
+        description: Whether to run the load test in production or staging
+        required: true
+        type: boolean
+        default: false
+      duration:
+        description: Duration of the test
+        required: true
+        type: string
+        default: 5m
+      playback-manifest-url:
+        description: URL of the playback manifest
+        type: string
+        required: false
+        default: ''
+      streamer-region:
+        description: Google Cloud region where to ingest from
+        type: string
+        required: true
+        default: us-central1
+      streamer-base-url:
+        description: RTMP ingest URL for the streamer
+        type: string
+        required: false
+      playback-region-viewers-json:
+        description: JSON object mapping Google Cloud region name to number of viewers. Must be multiples of 10.
+        type: string
+        required: true
+        default: '{"us-central1":20,"europe-west2":20,"asia-southeast1":20}'
+
+jobs:
+  load-test:
+    name: Run WebRTC load test
+    runs-on: ubuntu-latest
+    container:
+      image: livepeer/webrtc-load-tester:vg-chore-image-ci
+    steps:
+      - name: calculate-dates-and-times
+        id: timestamp
+        uses: lee-dohm/[email protected]
+        with:
+          format: ""
+
+      - name: Discord start notification
+        uses: Ilshidur/[email protected]
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+          DISCORD_USERNAME: ${{ github.triggering_actor }}
+          DISCORD_EMBEDS: >
+            [{
+              "title": "WebRTC load test starting",
+              "description": "A load test is starting (production=${{ inputs.production }}). Viewers: ${{ inputs.playback-region-viewers-json }}",
+              "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+              "timestamp": "${{ steps.timestamp.outputs.date }}",
+              "author": {
+                "name": "${{ github.triggering_actor }}",
+                "url": "${{ github.server_url }}/${{ github.triggering_actor }}",
+                "icon_url": "${{ github.server_url }}/${{ github.triggering_actor }}.png?s=32"
+              },
+              "fields": [{
+                "name": "Triggered timestamp",
+                "value": "${{ steps.timestamp.outputs.date }}",
+                "inline": true
+              }, {
+                "name": "Duration",
+                "value": "${{ inputs.duration }}",
+                "inline": true
+              }, {
+                "name": "Regional Viewers",
+                "value": "${{ inputs.playback-region-viewers-json }}",
+                "inline": true
+              }]
+            }]
+
+      - name: Prepare environment
+        id: env
+        run: |
+          STREAMER_BASE_URL_INPUT="${{ inputs.streamer-base-url }}"
+
+          if [ "${{ inputs.production }}" = "true" ]; then
+              API_SERVER="livepeer.com"
+              API_TOKEN="${{ secrets.LOAD_TEST_PROD_API_KEY }}"
+              STREAMER_BASE_URL="${STREAMER_BASE_URL_INPUT:-"rtmp://rtmp.livepeer.com/live/"}"
+              PLAYBACK_BASE_URL="https://lvpr.tv/"
+          else
+              API_SERVER="livepeer.monster"
+              API_TOKEN="${{ secrets.LOAD_TEST_STAGING_API_KEY }}"
+              STREAMER_BASE_URL="${STREAMER_BASE_URL_INPUT:-"rtmp://rtmp.livepeer.monster/live/"}"
+              PLAYBACK_BASE_URL="https://monster.lvpr.tv/"
+          fi
+          echo "::set-output name=api-server::${API_SERVER}"
+          echo "::set-output name=api-token::${API_TOKEN}"
+          echo "::set-output name=streamer-base-url::${STREAMER_BASE_URL}"
+          echo "::set-output name=playback-base-url::${PLAYBACK_BASE_URL}"
+
+      - name: Load Test
+        run: webrtc-load-tester orchestrator
+        env:
+          LT_WEBRTC_DURATION: ${{ inputs.duration }}
+          LT_WEBRTC_API_SERVER: ${{ steps.env.outputs.api-server }}
+          LT_WEBRTC_API_TOKEN: ${{ steps.env.outputs.api-token }}
+          LT_WEBRTC_STREAMER_REGION: ${{ inputs.streamer-region }}
+          LT_WEBRTC_STREAMER_BASE_URL: ${{ steps.env.outputs.streamer-base-url }}
+          LP_WEBRTC_STREAMER_INPUT_FILE: https://storage.googleapis.com/lp_testharness_assets/countdown_720p_30fps_2sGOP_noBframes_5min.mp4
+          LT_WEBRTC_PLAYBACK_BASE_URL: ${{ steps.env.outputs.playback-base-url }}
+          LT_WEBRTC_PLAYBACK_MANIFEST_URL: "${{ inputs.playback-manifest-url }}"
+          LT_WEBRTC_PLAYBACK_VIEWERS_PER_WORKER: 10
+          LT_WEBRTC_PLAYBACK_VIEWERS_PER_CPU: 2
+          LT_WEBRTC_PLAYBACK_MEMORY_PER_VIEWER_MIB: 400
+          LT_WEBRTC_PLAYBACK_REGION_VIEWERS_JSON: '${{ inputs.playback-region-viewers-json }}'
+          LT_WEBRTC_PLAYBACK_BASE_SCREENSHOT_FOLDER_OS: ${{ secrets.LOAD_TEST_SCREENSHOT_FOLDER_OS }}
+          LT_WEBRTC_GOOGLE_CREDENTIALS_JSON: '${{ secrets.LOAD_TEST_GOOGLE_CREDENTIALS_JSON }}'
+
+      - name: Discord finish notification
+        uses: Ilshidur/[email protected]
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+          DISCORD_USERNAME: ${{ github.triggering_actor }}
+          DISCORD_EMBEDS: >
+            [{
+              "title": " WebRTC load test finished successfully",
+              "description": "The load test (production=${{ inputs.production }}) has finished successfully",
+              "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+              "color": 24576,
+              "author": {
+                "name": "${{ github.triggering_actor }}",
+                "url": "${{ github.server_url }}/${{ github.triggering_actor }}",
+                "icon_url": "${{ github.server_url }}/${{ github.triggering_actor }}.png?s=32"
+              },
+              "fields": [{
+                "name": "Triggered timestamp",
+                "value": "${{ steps.timestamp.outputs.date }}",
+                "inline": true
+              }, {
+                "name": "Duration",
+                "value": "${{ inputs.duration }}",
+                "inline": true
+              }, {
+                "name": "Regional Viewers",
+                "value": ${{ toJSON(inputs.playback-region-viewers-json) }},
+                "inline": true
+              }]
+            }]
+
+      - name: Discord failure notification
+        uses: Ilshidur/[email protected]
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+          DISCORD_USERNAME: ${{ github.triggering_actor }}
+          DISCORD_EMBEDS: >
+            [{
+              "title": "WebRTC load test has failed! (production=${{ inputs.production }})",
+              "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+              "color": 8388608,
+              "author": {
+                "name": "${{ github.triggering_actor }}",
+                "url": "${{ github.server_url }}/${{ github.triggering_actor }}",
+                "icon_url": "${{ github.server_url }}/${{ github.triggering_actor }}.png?s=32"
+              },
+              "fields": [{
+                "name": "Triggered timestamp",
+                "value": "${{ steps.timestamp.outputs.date }}",
+                "inline": true
+              }, {
+                "name": "Duration",
+                "value": "${{ inputs.duration }}",
+                "inline": true
+              }, {
+                "name": "Regional Viewers",
+                "value": ${{ toJSON(inputs.playback-region-viewers-json) }},
+                "inline": true
+              }]
+            }]
+
+      - name: Discord cancel notification
+        uses: Ilshidur/[email protected]
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+          DISCORD_USERNAME: ${{ github.triggering_actor }}
+          DISCORD_EMBEDS: >
+            [{
+              "title": "WebRTC load test was cancelled! (production=${{ inputs.production }})",
+              "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+              "color": 11778048,
+              "author": {
+                "name": "${{ github.triggering_actor }}",
+                "url": "${{ github.server_url }}/${{ github.triggering_actor }}",
+                "icon_url": "${{ github.server_url }}/${{ github.triggering_actor }}.png?s=32"
+              },
+              "fields": [{
+                "name": "Triggered timestamp",
+                "value": "${{ steps.timestamp.outputs.date }}",
+                "inline": true
+              }, {
+                "name": "Duration",
+                "value": "${{ inputs.duration }}",
+                "inline": true
+              }, {
+                "name": "Regional Viewers",
+                "value": ${{ toJSON(inputs.playback-region-viewers-json) }},
+                "inline": true
+              }]
+            }]
diff --git a/cmd/webrtc-load-tester/roles/orchestrator.go b/cmd/webrtc-load-tester/roles/orchestrator.go
@@ -111,6 +111,7 @@ func Orchestrator() {
 func initClients(cliFlags loadTestArguments) {
 	studioApi = api.NewAPIClient(api.ClientOptions{Server: cliFlags.APIServer, AccessToken: cliFlags.APIToken})
 
+	glog.Infof("Creating Google client with creds: %q", cliFlags.GoogleCredentialsJSON)
 	err := gcloud.InitClients(context.Background(), cliFlags.GoogleCredentialsJSON, cliFlags.GoogleProjectID)
 	if err != nil {
 		glog.Errorf("Error initializing cloud run client: %v", err)

diff --git a/cmd/webrtc-load-tester/utils/command.go b/cmd/webrtc-load-tester/utils/command.go
@@ -32,6 +32,7 @@ func ParseFlags(registerVars func(*flag.FlagSet)) {
 		ff.WithConfigFileFlag("config"),
 		ff.WithConfigFileParser(ff.PlainParser),
 		ff.WithEnvVarPrefix("LT_WEBRTC"),
+		ff.WithEnvVarIgnoreCommas(true),
 	)
 	if err != nil {
 		glog.Errorf("Error parsing args: %v", err)
@@ -56,6 +57,8 @@ func JSONVarFlag(fs *flag.FlagSet, dest interface{}, name, defaultValue, usage s
 		panic(err)
 	}
 	fs.Func(name, usage, func(s string) error {
+		glog.Infof("Setting %s to: %q", name, s)
+
 		// Clear any previously set value, including the default above
 		destVal := reflect.ValueOf(dest).Elem()
 		destVal.Set(reflect.Zero(destVal.Type()))

diff --git a/go.mod b/go.mod
@@ -55,7 +55,7 @@ require (
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.3.1 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -112,6 +112,7 @@ require (
 	cloud.google.com/go/run v1.3.1
 	github.com/chromedp/chromedp v0.9.3
 	github.com/livepeer/catalyst-api v0.1.1
+	github.com/livepeer/go-tools v0.3.0
 )
 
 require (
@@ -173,7 +174,6 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	github.com/livepeer/go-tools v0.3.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/miekg/dns v1.1.50 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect

diff --git a/go.sum b/go.sum
@@ -554,8 +554,6 @@ github.com/livepeer/catalyst-api v0.1.1 h1:WP4rHH88b+lsxo33wPCjl0yvqVDNyxkleZH1s
 github.com/livepeer/catalyst-api v0.1.1/go.mod h1:d6XPE9ehhCutWhCqqcmlYqQa+e9bf3Ke92x+gRZlzoQ=
 github.com/livepeer/go-api-client v0.4.13 h1:H0kY0boMKRTpkHGXlIGMgj0cefEYw+KUPZwx4YLjq4A=
 github.com/livepeer/go-api-client v0.4.13/go.mod h1:Jdb+RI7JyzEZOHd1GUuKofwFDKMO/btTa80SdpUpYQw=
-github.com/livepeer/go-livepeer v0.7.1 h1:ubEUZYm5r0LYgQ5i+Sy8g1HNJqjVgmhcRyGiYds8648=
-github.com/livepeer/go-livepeer v0.7.1/go.mod h1:d6qTStiNmXTQ/5YLB9fhzgDV9MdXg3KmqESQpur2Ak0=
 github.com/livepeer/go-livepeer v0.7.2-0.20231110152159-b17a70dfe719 h1:468kFmwQFaI00eNCLL8qA5XuIBMwqqVgKEXvqS7msa8=
 github.com/livepeer/go-livepeer v0.7.2-0.20231110152159-b17a70dfe719/go.mod h1:d6qTStiNmXTQ/5YLB9fhzgDV9MdXg3KmqESQpur2Ak0=
 github.com/livepeer/go-tools v0.3.0 h1:xK0mJyPWWyvj9Oi9nfLglhCtk0KM8883WB7VO1oPF8g=