run_all.yml

name: Run Benchmarks

on:
  workflow_dispatch:
    inputs:
      root_dir:
        description: "Enter the root directory for benchmarks"
        required: true
        default: "/root/benchmarks/"
      date:
        description: "Enter a date (optional, e.g. 20240101)"
        required: false
        default: ""
      clients:
        description: 'Enter clients to use (e.g., [nethermind,geth,reth,erigon,besu])'
        required: true
        default: '[nethermind,geth,reth,erigon,besu]'
      sizes:
        description: "Enter sizes (e.g., [500,1000])"
        required: false
        default: "[500,1000]"
      repetitions:
        description: "Enter the number of repetitions"
        required: true
        default: 8
  schedule:
    - cron: '0 0 * * 1,3,5'

jobs:
  setup-environment:
    runs-on: ubuntu-latest
    env:
      ROOT_DIR: ${{ github.event.inputs.root_dir || '/root/benchmarks/' }}
      DATE: ${{ github.event.inputs.date || '' }}
      CLIENTS: ${{ github.event.inputs.clients || '["nethermind","geth","reth","erigon","besu"]' }}
      SIZES: ${{ github.event.inputs.sizes || '[500,1000]' }}
      REPETITIONS: ${{ github.event.inputs.repetitions || 8 }}
    outputs:
      date: ${{ steps.set-date.outputs.date }}
      root_dir: ${{ steps.set-date.outputs.root_dir }}
      clients: ${{ steps.set-date.outputs.clients }}
      sizes: ${{ steps.set-date.outputs.sizes }}
      repetitions: ${{ steps.set-date.outputs.repetitions }}
    steps:
      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.PRIVKEY }}
      - name: Set DATE variable
        id: set-date
        run: |
          if [ -z "${{ env.DATE }}" ]; then
            DATE=$(date +%Y%m%d)
          else
            DATE=${{ env.DATE }}
          fi
          echo "date=${DATE}" >> $GITHUB_OUTPUT
          echo "root_dir=${{ env.ROOT_DIR }}" >> $GITHUB_OUTPUT
          echo "clients=${{ env.CLIENTS }}" >> $GITHUB_OUTPUT
          echo "sizes=${{ env.SIZES }}" >> $GITHUB_OUTPUT
          echo "repetitions=${{ env.REPETITIONS }}" >> $GITHUB_OUTPUT
      - name: Clone benchmarks repos
        run: |
          BRANCH_BENCHMARKS_DATA_RESULT=main
          BRANCH_GAS_BENCHMARKS=rc-1
          BRANCH_BURNTPIX_BENCHMARKS=burnt-rc-1
          BRANCH_GENESIS_INIT_BENCHMARKS=main
          
          ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF
            set +e
            mkdir -p "${{ env.ROOT_DIR }}"
            cd "${{ env.ROOT_DIR }}"
            git clone -b $BRANCH_BENCHMARKS_DATA_RESULT https://${{ secrets.GIT_TOEKN_PREFIX }}@github.com/OpenFusionist/benchmarks-data-results.git benchmarks-data-result
            git clone -b $BRANCH_GAS_BENCHMARKS https://github.com/OpenFusionist/gas-benchmarks.git gas-benchmarks
            git clone -b $BRANCH_BURNTPIX_BENCHMARKS https://github.com/OpenFusionist/gas-benchmarks.git burntpix-benchmarks
            git clone -b $BRANCH_GENESIS_INIT_BENCHMARKS https://github.com/OpenFusionist/genesis-init-benchmarks.git genesis-init-benchmarks
            cd "${{ env.ROOT_DIR }}/benchmarks-data-result"
            git switch $BRANCH_BENCHMARKS_DATA_RESULT
            cd "${{ env.ROOT_DIR }}/gas-benchmarks"
            git switch $BRANCH_GAS_BENCHMARKS
            cd "${{ env.ROOT_DIR }}/burntpix-benchmarks"
            git switch $BRANCH_BURNTPIX_BENCHMARKS
            cd "${{ env.ROOT_DIR }}/genesis-init-benchmarks"
            git switch $BRANCH_GENESIS_INIT_BENCHMARKS
            set -e
          EOF
      - name: Stop existing benchmarks and prepare environment
        run: |
          ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF
            set +e
            cd "${{ env.ROOT_DIR }}/benchmarks-data-result"
            git pull
            ./stop.sh
            python3 ./update_time.py "${{ env.DATE }}"
            cat ./date.json
            set -e
          EOF

  generate-matrix:
    runs-on: ubuntu-latest
    needs: setup-environment
    outputs:
      matrix: ${{ steps.generate-matrix.outputs.matrix }}
      date: ${{ needs.setup-environment.outputs.date }}
      root_dir: ${{ needs.setup-environment.outputs.root_dir }}
      clients: ${{ needs.setup-environment.outputs.clients }}
      sizes: ${{ needs.setup-environment.outputs.sizes }}
      repetitions: ${{ needs.setup-environment.outputs.repetitions }}
    steps:
      - name: Check Date
        id: check-date
        run: |
          if [ -z "${{ needs.setup-environment.outputs.date }}" ]; then
            echo "Date is empty, exiting the job."
            exit 1
          fi

      - name: Generate Matrix
        id: generate-matrix
        run: |
          CLIENTS='${{ needs.setup-environment.outputs.clients }}'
          SIZES='${{ needs.setup-environment.outputs.sizes }}'
          echo "CLIENTS=${CLIENTS}"
          echo "SIZES=${SIZES}"

          CLIENTS_JSON=$(echo "$CLIENTS" | sed 's/[][]//g' | jq -R 'split(",")')
          SIZES_JSON=$(echo "$SIZES" | sed 's/[][]//g' | jq -R 'split(",") | map(tonumber)')

          MATRIX_CLIENTS=$(echo '{}' | jq -c --argjson clients "$CLIENTS_JSON" '.Client = $clients')
          MATRIX=$(echo $MATRIX_CLIENTS | jq -c --argjson sizes "$SIZES_JSON" '.Size = $sizes')

          echo "Updated MATRIX=${MATRIX}"
          echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
          
  run-gas-benchmarks:
    runs-on: ubuntu-latest
    needs: generate-matrix
    outputs:
      matrix: ${{ needs.generate-matrix.outputs.matrix }}
      date: ${{ needs.generate-matrix.outputs.date }}
      root_dir: ${{ needs.generate-matrix.outputs.root_dir }}
      clients: ${{ needs.generate-matrix.outputs.clients }}
      sizes: ${{ needs.generate-matrix.outputs.sizes }}
      repetitions: ${{ needs.generate-matrix.outputs.repetitions }}
    strategy:
      max-parallel: 1
      matrix:
        client: ${{ fromJson(needs.generate-matrix.outputs.matrix).client }}
      fail-fast: false
    steps:
      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.PRIVKEY }}
      - name: Run benchmarks
        run: |
          CLIENT=${{ matrix.client }}
          REPETITIONS=${{ needs.generate-matrix.outputs.repetitions }}
          ROOT_DIR=${{ needs.generate-matrix.outputs.root_dir }}
          DATE=${{ needs.generate-matrix.outputs.date }}

          ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF
            set +e
            echo "=== Run gas benchmarks for client: $CLIENT and date: $DATE ==="
            cd "$ROOT_DIR/gas-benchmarks"
            git pull
            cp ../benchmarks-data-result/images.yaml .
            ./run.sh -t "tests/" -w "warmup/warmup-1000bl-16wi-24tx.txt" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE"
            python3 report_metadata.py "$DATE"
            cp "results-$DATE/reports/index.html" "../benchmarks-data-result/gas/$DATE.html"
            cp "results-$DATE/reports/result.json" "../benchmarks-data-result/gas/$DATE.json"
            cp "results-$DATE/reports/metadata.json" "../benchmarks-data-result/gas/$DATE.metadata.json"
            set -e
          EOF

  run-genesis-init-speed-benchmarks:
    runs-on: ubuntu-latest
    needs: run-gas-benchmarks
    outputs:
      matrix: ${{ needs.run-gas-benchmarks.outputs.matrix }}
      date: ${{ needs.run-gas-benchmarks.outputs.date }}
      root_dir: ${{ needs.run-gas-benchmarks.outputs.root_dir }}
      clients: ${{ needs.run-gas-benchmarks.outputs.clients }}
      sizes: ${{ needs.run-gas-benchmarks.outputs.sizes }}
      repetitions: ${{ needs.run-gas-benchmarks.outputs.repetitions }}
    strategy:
      max-parallel: 1
      matrix: ${{ fromJson(needs.run-gas-benchmarks.outputs.matrix) }}
      fail-fast: false
    steps:
      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.PRIVKEY }}
      - name: Run genesis init speed benchmarks
        run: |
          CLIENT=${{ matrix.client }}
          SIZE=${{ matrix.size }}
          REPETITIONS=${{ needs.run-gas-benchmarks.outputs.repetitions }}
          ROOT_DIR=${{ needs.run-gas-benchmarks.outputs.root_dir }}
          DATE=${{ needs.run-gas-benchmarks.outputs.date }}

          ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF
            set +e
            echo "=== Run genesis init speed benchmarks for client: $CLIENT with size: $SIZE and date: $DATE ==="
            cd "$ROOT_DIR/genesis-init-benchmarks"
            git pull
            cp ../benchmarks-data-result/images.yaml .
            ./runSpeed.sh -t "tests/" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE/speed" -s "$SIZE"
            python3 report_metadata.py "$DATE" speed
            cp "results-$DATE/speed/reports/speed.html" "../benchmarks-data-result/genesis-data/$DATE.speed.html"
            cp "results-$DATE/speed/reports/speed.json" "../benchmarks-data-result/genesis-data/$DATE.speed.json"
            cp "results-$DATE/speed/reports/metadata.json" "../benchmarks-data-result/genesis-data/$DATE.metadata.json"
            set -e
          EOF

  run-genesis-init-memory-benchmarks:
    runs-on: ubuntu-latest
    needs: run-genesis-init-speed-benchmarks
    outputs:
      matrix: ${{ needs.run-genesis-init-speed-benchmarks.outputs.matrix }}
      date: ${{ needs.run-genesis-init-speed-benchmarks.outputs.date }}
      root_dir: ${{ needs.run-genesis-init-speed-benchmarks.outputs.root_dir }}
      clients: ${{ needs.run-genesis-init-speed-benchmarks.outputs.clients }}
      sizes: ${{ needs.run-genesis-init-speed-benchmarks.outputs.sizes }}
      repetitions: ${{ needs.run-genesis-init-speed-benchmarks.outputs.repetitions }}
    strategy:
      max-parallel: 1
      matrix: ${{ fromJson(needs.run-genesis-init-speed-benchmarks.outputs.matrix) }}
      fail-fast: false
    steps:
      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.PRIVKEY }}
      - name: Run genesis init memory benchmarks
        run: |
          CLIENT=${{ matrix.client }}
          SIZE=${{ matrix.size }}
          REPETITIONS=${{ needs.run-genesis-init-speed-benchmarks.outputs.repetitions }}
          ROOT_DIR=${{ needs.run-genesis-init-speed-benchmarks.outputs.root_dir }}
          DATE=${{ needs.run-genesis-init-speed-benchmarks.outputs.date }}

          ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF
            set +e
            echo "=== Run genesis init memory benchmarks for client: $CLIENT with size: $SIZE and date: $DATE ==="
            cd "$ROOT_DIR/genesis-init-benchmarks"
            git pull
            cp ../benchmarks-data-result/images.yaml .
            ./runMemory.sh -t "tests/" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE/memory" -s "$SIZE"
            python3 report_metadata.py "$DATE" memory
            cp "results-$DATE/memory/reports/memory.html" "../benchmarks-data-result/genesis-data/$DATE.memory.html"
            cp "results-$DATE/memory/reports/memory.json" "../benchmarks-data-result/genesis-data/$DATE.memory.json"
            cp "results-$DATE/memory/reports/metadata.json" "../benchmarks-data-result/genesis-data/$DATE.metadata.json"
            python3 merge_result.py "$DATE"
            cp "results-$DATE/result.json" "../benchmarks-data-result/genesis-data/$DATE.json"
            set -e
          EOF

  run-burntpix-benchmarks:
    runs-on: ubuntu-latest
    needs: run-genesis-init-memory-benchmarks
    outputs:
      matrix: ${{ needs.run-genesis-init-memory-benchmarks.outputs.matrix }}
      date: ${{ needs.run-genesis-init-memory-benchmarks.outputs.date }}
      root_dir: ${{ needs.run-genesis-init-memory-benchmarks.outputs.root_dir }}
      clients: ${{ needs.run-genesis-init-memory-benchmarks.outputs.clients }}
      sizes: ${{ needs.run-genesis-init-memory-benchmarks.outputs.sizes }}
      repetitions: ${{ needs.run-genesis-init-memory-benchmarks.outputs.repetitions }}
    steps:
      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.PRIVKEY }}
      - name: Run burntpix benchmarks
        run: |
          # Use const due to can't handle ${{ needs.run-genesis-init-memory-benchmarks.outputs.clients }}
          CLIENT="nethermind,geth,reth,erigon,besu"
          REPETITIONS=${{ needs.run-genesis-init-memory-benchmarks.outputs.repetitions }}
          ROOT_DIR=${{ needs.run-genesis-init-memory-benchmarks.outputs.root_dir }}
          DATE=${{ needs.run-genesis-init-memory-benchmarks.outputs.date }}

          ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF
            set +e
            echo "=== Run burntpix benchmarks for client: $CLIENT with date: $DATE ==="
            cd "$ROOT_DIR/burntpix-benchmarks"
            git pull
            cp ../benchmarks-data-result/images.yaml .
            ./run.sh -t "testburnt/" -w "./testburntwarm/warm.txt" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE"
            python3 report_metadata.py "$DATE"
            cp "results-$DATE/reports/index.html" "../benchmarks-data-result/burntpix/$DATE.html"
            cp "results-$DATE/reports/result.json" "../benchmarks-data-result/burntpix/$DATE.json"
            cp "results-$DATE/reports/metadata.json" "../benchmarks-data-result/burntpix/$DATE.metadata.json"
            set -e
          EOF

  push-results:
    runs-on: ubuntu-latest
    needs: run-burntpix-benchmarks
    steps:
      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.PRIVKEY }}
      - name: Push results
        run: |
          ROOT_DIR=${{ needs.run-burntpix-benchmarks.outputs.root_dir }}

          ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF
            set +e
            cd "$ROOT_DIR/benchmarks-data-result"
            git status
            ./push.sh
            set -e
          EOF