From cebdbeef9019a4e2e9bbeb859581f08d23952f46 Mon Sep 17 00:00:00 2001
From: Doug <4741454+douglance@users.noreply.github.com>
Date: Tue, 20 Aug 2024 10:40:51 -0400
Subject: [PATCH] ci: add monitoring test matrix (#1848)

---
 .github/workflows/assertion-monitor.yml       | 19 ++++++
 .github/workflows/batch-poster-monitor.yml    | 19 ++++++
 .../workflows/core-batch-poster-monitor.yml   | 61 -----------------
 .github/workflows/core-retryable-monitor.yml  | 58 ----------------
 .github/workflows/monitor-config.json         | 30 ++++++++
 .github/workflows/monitoring.yml              | 68 +++++++++++++++++++
 .../workflows/orbit-batch-poster-monitor.yml  | 56 ---------------
 .github/workflows/orbit-retryable-monitor.yml | 56 ---------------
 .github/workflows/retryable-monitor.yml       | 19 ++++++
 9 files changed, 155 insertions(+), 231 deletions(-)
 create mode 100644 .github/workflows/assertion-monitor.yml
 create mode 100644 .github/workflows/batch-poster-monitor.yml
 delete mode 100644 .github/workflows/core-batch-poster-monitor.yml
 delete mode 100644 .github/workflows/core-retryable-monitor.yml
 create mode 100644 .github/workflows/monitor-config.json
 create mode 100644 .github/workflows/monitoring.yml
 delete mode 100644 .github/workflows/orbit-batch-poster-monitor.yml
 delete mode 100644 .github/workflows/orbit-retryable-monitor.yml
 create mode 100644 .github/workflows/retryable-monitor.yml

diff --git a/.github/workflows/assertion-monitor.yml b/.github/workflows/assertion-monitor.yml
new file mode 100644
index 0000000000..a29ceae4ae
--- /dev/null
+++ b/.github/workflows/assertion-monitor.yml
@@ -0,0 +1,19 @@
+name: Assertion Monitor
+
+on:
+  push:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 */6 * * *"  # Run every 6 hours
+
+jobs:
+  run-monitoring:
+    name: Assertion Monitor (${{ matrix.chain }})
+    strategy:
+      matrix:
+        chain: [core, orbit]
+    uses: ./.github/workflows/monitoring.yml
+    with:
+      chain: ${{ matrix.chain }}
+      monitor: assertion
+    secrets: inherit
diff --git a/.github/workflows/batch-poster-monitor.yml b/.github/workflows/batch-poster-monitor.yml
new file mode 100644
index 0000000000..8d9af811f0
--- /dev/null
+++ b/.github/workflows/batch-poster-monitor.yml
@@ -0,0 +1,19 @@
+name: Batch Poster Monitor
+
+on:
+  push:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 */6 * * *"  # Run every 6 hours
+
+jobs:
+  run-monitoring:
+    name: Batch Poster Monitor (${{ matrix.chain }})
+    strategy:
+      matrix:
+        chain: [core, orbit]
+    uses: ./.github/workflows/monitoring.yml
+    with:
+      chain: ${{ matrix.chain }}
+      monitor: batch-poster
+    secrets: inherit
diff --git a/.github/workflows/core-batch-poster-monitor.yml b/.github/workflows/core-batch-poster-monitor.yml
deleted file mode 100644
index bdd0b6f876..0000000000
--- a/.github/workflows/core-batch-poster-monitor.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-name: Monitor Core Batch Posting
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Run every 6 hours
-    - cron: "0 */6 * * *" 
-
-env:
-  NEXT_PUBLIC_INFURA_KEY: ${{ secrets.NEXT_PUBLIC_INFURA_KEY }}
-  BATCH_POSTER_MONITORING_SLACK_TOKEN: ${{ secrets.CORE_CHAIN_BATCH_POSTER_MONITORING_SLACK_TOKEN }}
-  BATCH_POSTER_MONITORING_SLACK_CHANNEL: ${{ secrets.CORE_CHAIN_BATCH_POSTER_MONITORING_SLACK_CHANNEL }}
-  NOVA_MONITOR_RPC_URL: ${{ secrets.NOVA_MONITOR_RPC_URL }}
-  ARB_ONE_MONITOR_RPC_URL: ${{ secrets.ARB_ONE_MONITOR_RPC_URL }}
-
-jobs:
-  run-alerting:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout Bridge
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-token-bridge
-
-    - name: Install node_modules
-      uses: OffchainLabs/actions/node-modules/install@main
-
-    - name: Generate chains JSON
-      run: yarn workspace arb-token-bridge-ui generateCoreChainsToMonitor
-      env:
-        NOVA_MONITOR_RPC_URL: ${{ secrets.NOVA_MONITOR_RPC_URL }}
-        ARB_ONE_MONITOR_RPC_URL: ${{ secrets.ARB_ONE_MONITOR_RPC_URL }}
-        BATCH_POSTER_MONITORING: true
-
-    - name: Checkout Arbitrum Monitoring repository
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-monitoring
-        path: arbitrum-monitoring
-
-    - name: Copy chains JSON to Arbitrum Monitoring
-      run: cp ./packages/arb-token-bridge-ui/public/__auto-generated-core-chains.json ./arbitrum-monitoring/packages/batch-poster-monitor/config.json
-   
-    - name: Setup Node for Arbitrum Monitoring
-      uses: actions/setup-node@v4
-      with:
-        node-version: latest
-
-    - name: Install dependencies for Arbitrum Monitoring
-      run: cd ./arbitrum-monitoring && yarn install
-
-    - name: Run alerting command
-      run: cd ./arbitrum-monitoring && yarn batch-poster-monitor --enableAlerting
-      env:
-        NODE_ENV: "CI"
-        RETRYABLE_MONITORING_SLACK_TOKEN: ${{ secrets.CORE_CHAIN_BATCH_POSTER_MONITORING_SLACK_TOKEN }}
-        RETRYABLE_MONITORING_SLACK_CHANNEL: ${{ secrets.CORE_CHAIN_BATCH_POSTER_MONITORING_SLACK_CHANNEL }}
-
-    - name: Clean up
-      run: rm ./packages/arb-token-bridge-ui/public/__auto-generated-core-chains.json
diff --git a/.github/workflows/core-retryable-monitor.yml b/.github/workflows/core-retryable-monitor.yml
deleted file mode 100644
index 7ca21eedb0..0000000000
--- a/.github/workflows/core-retryable-monitor.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-name: Monitor Core Retryables
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Run once a day at 08:03am GMT (https://github.com/OffchainLabs/arb-monitoring/blob/master/.github/workflows/retryables.yml#L5)
-    - cron: "3 8 * * *" 
-
-env:
-  NEXT_PUBLIC_INFURA_KEY: ${{ secrets.NEXT_PUBLIC_INFURA_KEY }}
-  RETRYABLE_MONITORING_SLACK_TOKEN: ${{ secrets.CORE_CHAIN_RETRYABLE_MONITORING_SLACK_TOKEN }}
-  RETRYABLE_MONITORING_SLACK_CHANNEL: ${{ secrets.CORE_CHAIN_RETRYABLE_MONITORING_SLACK_CHANNEL }}
-  NOVA_MONITOR_RPC_URL: ${{ secrets.NOVA_MONITOR_RPC_URL }}
-
-jobs:
-  run-alerting:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout Bridge
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-token-bridge
-
-    - name: Install node_modules
-      uses: OffchainLabs/actions/node-modules/install@main
-
-    - name: Generate chains JSON
-      run: yarn workspace arb-token-bridge-ui generateCoreChainsToMonitor
-      env:
-        NOVA_MONITOR_RPC_URL: ${{ secrets.NOVA_MONITOR_RPC_URL }}
-
-    - name: Checkout Arbitrum Monitoring repository
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-monitoring
-        path: arbitrum-monitoring
-
-    - name: Copy chains JSON to Arbitrum Monitoring
-      run: cp ./packages/arb-token-bridge-ui/public/__auto-generated-core-chains.json ./arbitrum-monitoring/packages/retryable-monitor/config.json
-   
-    - name: Setup Node for Arbitrum Monitoring
-      uses: actions/setup-node@v4
-      with:
-        node-version: latest
-
-    - name: Install dependencies for Arbitrum Monitoring
-      run: cd ./arbitrum-monitoring && yarn install
-
-    - name: Run alerting command
-      run: cd ./arbitrum-monitoring && yarn retryable-monitor --enableAlerting
-      env:
-        NODE_ENV: "CI"
-        RETRYABLE_MONITORING_SLACK_TOKEN: ${{ secrets.CORE_CHAIN_RETRYABLE_MONITORING_SLACK_TOKEN }}
-        RETRYABLE_MONITORING_SLACK_CHANNEL: ${{ secrets.CORE_CHAIN_RETRYABLE_MONITORING_SLACK_CHANNEL }}
-
-    - name: Clean up
-      run: rm ./packages/arb-token-bridge-ui/public/__auto-generated-core-chains.json
diff --git a/.github/workflows/monitor-config.json b/.github/workflows/monitor-config.json
new file mode 100644
index 0000000000..6f35882143
--- /dev/null
+++ b/.github/workflows/monitor-config.json
@@ -0,0 +1,30 @@
+{
+  "core": {
+    "generateCommand": "generateCoreChainsToMonitor",
+    "configFile": "__auto-generated-core-chains.json",
+    "slackTokens": {
+      "assertion": "CORE_CHAIN_ASSERTION_MONITORING_SLACK_TOKEN",
+      "batch-poster": "CORE_CHAIN_BATCH_POSTER_MONITORING_SLACK_TOKEN",
+      "retryable": "CORE_CHAIN_RETRYABLE_MONITORING_SLACK_TOKEN"
+    },
+    "slackChannels": {
+      "assertion": "CORE_CHAIN_ASSERTION_MONITORING_SLACK_CHANNEL",
+      "batch-poster": "CORE_CHAIN_BATCH_POSTER_MONITORING_SLACK_CHANNEL",
+      "retryable": "CORE_CHAIN_RETRYABLE_MONITORING_SLACK_CHANNEL"
+    }
+  },
+  "orbit": {
+    "generateCommand": "generateOrbitChainsToMonitor",
+    "configFile": "__auto-generated-orbit-chains.json",
+    "slackTokens": {
+      "assertion": "ORBIT_CHAIN_ASSERTION_MONITORING_SLACK_TOKEN",
+      "batch-poster": "ORBIT_CHAIN_BATCH_POSTER_MONITORING_SLACK_TOKEN",
+      "retryable": "ORBIT_RETRYABLE_MONITORING_SLACK_TOKEN"
+    },
+    "slackChannels": {
+      "assertion": "ORBIT_CHAIN_ASSERTION_MONITORING_SLACK_CHANNEL",
+      "batch-poster": "ORBIT_CHAIN_BATCH_POSTER_MONITORING_SLACK_CHANNEL",
+      "retryable": "ORBIT_RETRYABLE_MONITORING_SLACK_CHANNEL"
+    }
+  }
+}
diff --git a/.github/workflows/monitoring.yml b/.github/workflows/monitoring.yml
new file mode 100644
index 0000000000..bf33675ba9
--- /dev/null
+++ b/.github/workflows/monitoring.yml
@@ -0,0 +1,68 @@
+name: Reusable Monitoring Workflow
+
+on:
+  workflow_call:
+    inputs:
+      chain:
+        required: true
+        type: string
+      monitor:
+        required: true
+        type: string
+
+env:
+  NEXT_PUBLIC_INFURA_KEY: ${{ secrets.NEXT_PUBLIC_INFURA_KEY }}
+  NOVA_MONITOR_RPC_URL: ${{ secrets.NOVA_MONITOR_RPC_URL }}
+  ARB_ONE_MONITOR_RPC_URL: ${{ secrets.ARB_ONE_MONITOR_RPC_URL }}
+  NODE_ENV: "CI"
+
+jobs:
+  run-monitoring:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repositories
+      uses: actions/checkout@v4
+      with:
+        repository: OffchainLabs/arbitrum-token-bridge
+
+    - name: Install node_modules
+      uses: OffchainLabs/actions/node-modules/install@main
+
+    - name: Checkout Arbitrum Monitoring repository
+      uses: actions/checkout@v4
+      with:
+        repository: OffchainLabs/arbitrum-monitoring
+        path: arbitrum-monitoring
+
+    - name: Setup Node and install dependencies
+      uses: actions/setup-node@v4
+      with:
+        node-version: latest
+    - run: cd ./arbitrum-monitoring && yarn install
+
+    - name: Load configuration
+      id: config
+      run: |
+        CONFIG=$(cat .github/workflows/monitor-config.json | jq -r '.${{ inputs.chain }}')
+        echo "generate_command=$(echo $CONFIG | jq -r '.generateCommand')" >> $GITHUB_OUTPUT
+        echo "config_file=$(echo $CONFIG | jq -r '.configFile')" >> $GITHUB_OUTPUT
+        echo "slack_token=$(echo $CONFIG | jq -r '.slackTokens.${{ inputs.monitor }}')" >> $GITHUB_OUTPUT
+        echo "slack_channel=$(echo $CONFIG | jq -r '.slackChannels.${{ inputs.monitor }}')" >> $GITHUB_OUTPUT
+
+    - name: Generate chains JSON
+      run: yarn workspace arb-token-bridge-ui ${{ steps.config.outputs.generate_command }}
+      env:
+        BATCH_POSTER_MONITORING: ${{ inputs.monitor == 'batch-poster' }}
+
+    - name: Copy chains JSON to Arbitrum Monitoring
+      run: cp ./packages/arb-token-bridge-ui/public/${{ steps.config.outputs.config_file }} ./arbitrum-monitoring/packages/${{ inputs.monitor }}-monitor/config.json
+
+    - name: Run monitoring command
+      run: cd ./arbitrum-monitoring && yarn ${{ inputs.monitor }}-monitor --enableAlerting
+      env:
+        RETRYABLE_MONITORING_SLACK_TOKEN: ${{ secrets[steps.config.outputs.slack_token] }}
+        RETRYABLE_MONITORING_SLACK_CHANNEL: ${{ secrets[steps.config.outputs.slack_channel] }}
+        BATCH_POSTER_MONITORING_SLACK_TOKEN: ${{ secrets[steps.config.outputs.slack_token] }}
+        BATCH_POSTER_MONITORING_SLACK_CHANNEL: ${{ secrets[steps.config.outputs.slack_channel] }}
+        ASSERTION_MONITORING_SLACK_TOKEN: ${{ secrets[steps.config.outputs.slack_token] }}
+        ASSERTION_MONITORING_SLACK_CHANNEL: ${{ secrets[steps.config.outputs.slack_channel] }}
diff --git a/.github/workflows/orbit-batch-poster-monitor.yml b/.github/workflows/orbit-batch-poster-monitor.yml
deleted file mode 100644
index e72bbed120..0000000000
--- a/.github/workflows/orbit-batch-poster-monitor.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-name: Monitor Orbit Batch Posting
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Run every 6 hours
-    - cron: "0 */6 * * *" 
-
-env:
-  NEXT_PUBLIC_INFURA_KEY: ${{ secrets.NEXT_PUBLIC_INFURA_KEY }}
-  BATCH_POSTER_MONITORING_SLACK_TOKEN: ${{ secrets.ORBIT_CHAIN_BATCH_POSTER_MONITORING_SLACK_TOKEN }}
-  BATCH_POSTER_MONITORING_SLACK_CHANNEL: ${{ secrets.ORBIT_CHAIN_BATCH_POSTER_MONITORING_SLACK_CHANNEL }}
-
-jobs:
-  run-alerting:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout Bridge
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-token-bridge
-
-    - name: Install node_modules
-      uses: OffchainLabs/actions/node-modules/install@main
-
-    - name: Generate chains JSON
-      run: yarn workspace arb-token-bridge-ui generateOrbitChainsToMonitor
-
-    - name: Checkout Arbitrum Monitoring repository
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-monitoring
-        path: arbitrum-monitoring
-
-    - name: Copy chains JSON to Arbitrum Monitoring
-      run: cp ./packages/arb-token-bridge-ui/public/__auto-generated-orbit-chains.json ./arbitrum-monitoring/packages/batch-poster-monitor/config.json
-   
-    - name: Setup Node for Arbitrum Monitoring
-      uses: actions/setup-node@v4
-      with:
-        node-version: latest
-
-    - name: Install dependencies for Arbitrum Monitoring
-      run: cd ./arbitrum-monitoring && yarn install
-
-    - name: Run alerting command
-      run: cd ./arbitrum-monitoring && yarn batch-poster-monitor --enableAlerting
-      env:
-        NODE_ENV: "CI"
-        BATCH_POSTER_MONITORING_SLACK_TOKEN: ${{ secrets.ORBIT_CHAIN_BATCH_POSTER_MONITORING_SLACK_TOKEN }}
-        BATCH_POSTER_MONITORING_SLACK_CHANNEL: ${{ secrets.ORBIT_CHAIN_BATCH_POSTER_MONITORING_SLACK_CHANNEL }}
-
-
-    - name: Clean up
-      run: rm ./packages/arb-token-bridge-ui/public/__auto-generated-orbit-chains.json
diff --git a/.github/workflows/orbit-retryable-monitor.yml b/.github/workflows/orbit-retryable-monitor.yml
deleted file mode 100644
index 84753b7f7a..0000000000
--- a/.github/workflows/orbit-retryable-monitor.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-name: Monitor Orbit Retryables
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Run once a day at 08:03am GMT (https://github.com/OffchainLabs/arb-monitoring/blob/master/.github/workflows/retryables.yml#L5)
-    - cron: "3 8 * * *" 
-
-env:
-  NEXT_PUBLIC_INFURA_KEY: ${{ secrets.NEXT_PUBLIC_INFURA_KEY }}
-  RETRYABLE_MONITORING_SLACK_TOKEN: ${{ secrets.ORBIT_RETRYABLE_MONITORING_SLACK_TOKEN }}
-  RETRYABLE_MONITORING_SLACK_CHANNEL: ${{ secrets.ORBIT_RETRYABLE_MONITORING_SLACK_CHANNEL }}
-
-jobs:
-  run-alerting:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout Bridge
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-token-bridge
-
-    - name: Install node_modules
-      uses: OffchainLabs/actions/node-modules/install@main
-
-    - name: Generate chains JSON
-      run: yarn workspace arb-token-bridge-ui generateOrbitChainsToMonitor
-
-    - name: Checkout Arbitrum Monitoring repository
-      uses: actions/checkout@v4
-      with:
-        repository: OffchainLabs/arbitrum-monitoring
-        path: arbitrum-monitoring
-
-    - name: Copy chains JSON to Arbitrum Monitoring
-      run: cp ./packages/arb-token-bridge-ui/public/__auto-generated-orbit-chains.json ./arbitrum-monitoring/packages/retryable-monitor/config.json
-   
-    - name: Setup Node for Arbitrum Monitoring
-      uses: actions/setup-node@v4
-      with:
-        node-version: latest
-
-    - name: Install dependencies for Arbitrum Monitoring
-      run: cd ./arbitrum-monitoring && yarn install
-
-    - name: Run alerting command
-      run: cd ./arbitrum-monitoring && yarn retryable-monitor --enableAlerting
-      env:
-        NODE_ENV: "CI"
-        RETRYABLE_MONITORING_SLACK_TOKEN: ${{ secrets.ORBIT_RETRYABLE_MONITORING_SLACK_TOKEN }}
-        RETRYABLE_MONITORING_SLACK_CHANNEL: ${{ secrets.ORBIT_RETRYABLE_MONITORING_SLACK_CHANNEL }}
-
-
-    - name: Clean up
-      run: rm ./packages/arb-token-bridge-ui/public/__auto-generated-orbit-chains.json
diff --git a/.github/workflows/retryable-monitor.yml b/.github/workflows/retryable-monitor.yml
new file mode 100644
index 0000000000..4b251bbbee
--- /dev/null
+++ b/.github/workflows/retryable-monitor.yml
@@ -0,0 +1,19 @@
+name: Retryable Monitor
+
+on:
+  push:
+  workflow_dispatch:
+  schedule:
+    - cron: "3 8 * * *" # Run once a day at 08:03am GMT
+
+jobs:
+  run-retryable-monitoring:
+    name: Retryables Monitor (${{ matrix.chain }})
+    strategy:
+      matrix:
+        chain: [core, orbit]
+    uses: ./.github/workflows/monitoring.yml
+    with:
+      chain: ${{ matrix.chain }}
+      monitor: retryable
+    secrets: inherit