Run Benchmarks #49
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Benchmarks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| root_dir: | |
| description: "Enter the root directory for benchmarks" | |
| required: true | |
| default: "/root/benchmarks/" | |
| date: | |
| description: "Enter a date (optional, e.g. 20240101)" | |
| required: false | |
| default: "" | |
| clients: | |
| description: "Enter clients to use (e.g., [nethermind,geth,reth,erigon,besu])" | |
| required: true | |
| default: "[nethermind,geth,reth,erigon,besu]" | |
| sizes: | |
| description: "Enter sizes (e.g., [1000])" | |
| required: false | |
| default: "[1000]" | |
| repetitions: | |
| description: "Enter the number of repetitions" | |
| required: true | |
| default: 8 | |
| schedule: | |
| - cron: "0 0 * * 6" | |
| jobs: | |
| setup-environment: | |
| runs-on: ubuntu-latest | |
| env: | |
| ROOT_DIR: ${{ github.event.inputs.root_dir || '/root/benchmarks/' }} | |
| DATE: ${{ github.event.inputs.date || '' }} | |
| CLIENTS: ${{ github.event.inputs.clients || '["nethermind","geth","reth","erigon","besu"]' }} | |
| SIZES: ${{ github.event.inputs.sizes || '[1000]' }} | |
| REPETITIONS: ${{ github.event.inputs.repetitions || 8 }} | |
| outputs: | |
| date: ${{ steps.set-date.outputs.date }} | |
| root_dir: ${{ steps.set-date.outputs.root_dir }} | |
| clients: ${{ steps.set-date.outputs.clients }} | |
| sizes: ${{ steps.set-date.outputs.sizes }} | |
| repetitions: ${{ steps.set-date.outputs.repetitions }} | |
| steps: | |
| - name: Setup SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.PRIVKEY }} | |
| - name: Set DATE variable | |
| id: set-date | |
| run: | | |
| if [ -z "${{ env.DATE }}" ]; then | |
| DATE=$(date +%Y%m%d) | |
| else | |
| DATE=${{ env.DATE }} | |
| fi | |
| echo "date=${DATE}" >> $GITHUB_OUTPUT | |
| echo "root_dir=${{ env.ROOT_DIR }}" >> $GITHUB_OUTPUT | |
| echo "clients=${{ env.CLIENTS }}" >> $GITHUB_OUTPUT | |
| echo "sizes=${{ env.SIZES }}" >> $GITHUB_OUTPUT | |
| echo "repetitions=${{ env.REPETITIONS }}" >> $GITHUB_OUTPUT | |
| - name: Clone benchmarks repos | |
| run: | | |
| BRANCH_BENCHMARKS_DATA_RESULT=main | |
| BRANCH_GAS_BENCHMARKS=rc-1 | |
| BRANCH_BURNTPIX_BENCHMARKS=burnt-rc-1 | |
| BRANCH_GENESIS_INIT_BENCHMARKS=main | |
| ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF | |
| set +e | |
| # Clear target directories only if they exist | |
| [ -d "${{ env.ROOT_DIR }}/benchmarks-data-result" ] && rm -rf "${{ env.ROOT_DIR }}/benchmarks-data-result" | |
| [ -d "${{ env.ROOT_DIR }}/gas-benchmarks" ] && rm -rf "${{ env.ROOT_DIR }}/gas-benchmarks" | |
| [ -d "${{ env.ROOT_DIR }}/burntpix-benchmarks" ] && rm -rf "${{ env.ROOT_DIR }}/burntpix-benchmarks" | |
| [ -d "${{ env.ROOT_DIR }}/genesis-init-benchmarks" ] && rm -rf "${{ env.ROOT_DIR }}/genesis-init-benchmarks" | |
| # Create root directory if it doesn't exist | |
| mkdir -p "${{ env.ROOT_DIR }}" | |
| cd "${{ env.ROOT_DIR }}" | |
| # Clone repositories | |
| git clone -b $BRANCH_BENCHMARKS_DATA_RESULT https://${{ secrets.GIT_TOKEN_PREFIX }}@github.com/OpenFusionist/benchmarks-data-results.git benchmarks-data-result | |
| git clone -b $BRANCH_GAS_BENCHMARKS https://github.com/OpenFusionist/gas-benchmarks.git gas-benchmarks | |
| git clone -b $BRANCH_BURNTPIX_BENCHMARKS https://github.com/OpenFusionist/gas-benchmarks.git burntpix-benchmarks | |
| git clone -b $BRANCH_GENESIS_INIT_BENCHMARKS https://github.com/OpenFusionist/genesis-init-benchmarks.git genesis-init-benchmarks | |
| cd "${{ env.ROOT_DIR }}/benchmarks-data-result" | |
| git switch $BRANCH_BENCHMARKS_DATA_RESULT | |
| cd "${{ env.ROOT_DIR }}/gas-benchmarks" | |
| git switch $BRANCH_GAS_BENCHMARKS | |
| cd "${{ env.ROOT_DIR }}/burntpix-benchmarks" | |
| git switch $BRANCH_BURNTPIX_BENCHMARKS | |
| cd "${{ env.ROOT_DIR }}/genesis-init-benchmarks" | |
| git switch $BRANCH_GENESIS_INIT_BENCHMARKS | |
| set -e | |
| EOF | |
| - name: Stop existing benchmarks and prepare environment | |
| run: | | |
| ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF | |
| set +e | |
| cd "${{ env.ROOT_DIR }}/benchmarks-data-result" | |
| git reset --hard | |
| git clean -fd | |
| git pull | |
| ./stop.sh | |
| python3 ./update_time.py "${{ env.DATE }}" | |
| cat ./date.json | |
| set -e | |
| EOF | |
| generate-matrix: | |
| runs-on: ubuntu-latest | |
| needs: setup-environment | |
| outputs: | |
| matrix: ${{ steps.generate-matrix.outputs.matrix }} | |
| date: ${{ needs.setup-environment.outputs.date }} | |
| root_dir: ${{ needs.setup-environment.outputs.root_dir }} | |
| clients: ${{ needs.setup-environment.outputs.clients }} | |
| sizes: ${{ needs.setup-environment.outputs.sizes }} | |
| repetitions: ${{ needs.setup-environment.outputs.repetitions }} | |
| steps: | |
| - name: Check Date | |
| id: check-date | |
| run: | | |
| if [ -z "${{ needs.setup-environment.outputs.date }}" ]; then | |
| echo "Date is empty, exiting the job." | |
| exit 1 | |
| fi | |
| - name: Generate Matrix | |
| id: generate-matrix | |
| run: | | |
| CLIENTS='${{ needs.setup-environment.outputs.clients }}' | |
| SIZES='${{ needs.setup-environment.outputs.sizes }}' | |
| echo "CLIENTS=${CLIENTS}" | |
| echo "SIZES=${SIZES}" | |
| CLIENTS_JSON=$(echo "$CLIENTS" | sed 's/[][]//g' | jq -R 'split(",")') | |
| SIZES_JSON=$(echo "$SIZES" | sed 's/[][]//g' | jq -R 'split(",") | map(tonumber)') | |
| MATRIX_CLIENTS=$(echo '{}' | jq -c --argjson clients "$CLIENTS_JSON" '.Client = $clients') | |
| MATRIX=$(echo $MATRIX_CLIENTS | jq -c --argjson sizes "$SIZES_JSON" '.Size = $sizes') | |
| echo "Updated MATRIX=${MATRIX}" | |
| echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT | |
| run-gas-benchmarks: | |
| runs-on: ubuntu-latest | |
| needs: generate-matrix | |
| outputs: | |
| matrix: ${{ needs.generate-matrix.outputs.matrix }} | |
| date: ${{ needs.generate-matrix.outputs.date }} | |
| root_dir: ${{ needs.generate-matrix.outputs.root_dir }} | |
| clients: ${{ needs.generate-matrix.outputs.clients }} | |
| sizes: ${{ needs.generate-matrix.outputs.sizes }} | |
| repetitions: ${{ needs.generate-matrix.outputs.repetitions }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| client: ${{ fromJson(needs.generate-matrix.outputs.matrix).client }} | |
| fail-fast: false | |
| steps: | |
| - name: Setup SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.PRIVKEY }} | |
| - name: Run benchmarks | |
| run: | | |
| CLIENT=${{ matrix.client }} | |
| REPETITIONS=${{ needs.generate-matrix.outputs.repetitions }} | |
| ROOT_DIR=${{ needs.generate-matrix.outputs.root_dir }} | |
| DATE=${{ needs.generate-matrix.outputs.date }} | |
| ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF | |
| set +e | |
| echo "=== Run gas benchmarks for client: $CLIENT and date: $DATE ===" | |
| cd "$ROOT_DIR/gas-benchmarks" | |
| git pull | |
| cp ../benchmarks-data-result/images.yaml . | |
| ./run.sh -t "tests/" -w "warmup/warmup-1000bl-16wi-24tx.txt" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE" | |
| python3 report_metadata.py "$DATE" | |
| cp "results-$DATE/reports/index.html" "../benchmarks-data-result/gas/$DATE.html" | |
| cp "results-$DATE/reports/result.json" "../benchmarks-data-result/gas/$DATE.json" | |
| cp "results-$DATE/reports/metadata.json" "../benchmarks-data-result/gas/$DATE.metadata.json" | |
| set -e | |
| EOF | |
| run-genesis-init-speed-benchmarks: | |
| runs-on: ubuntu-latest | |
| needs: run-gas-benchmarks | |
| outputs: | |
| matrix: ${{ needs.run-gas-benchmarks.outputs.matrix }} | |
| date: ${{ needs.run-gas-benchmarks.outputs.date }} | |
| root_dir: ${{ needs.run-gas-benchmarks.outputs.root_dir }} | |
| clients: ${{ needs.run-gas-benchmarks.outputs.clients }} | |
| sizes: ${{ needs.run-gas-benchmarks.outputs.sizes }} | |
| repetitions: ${{ needs.run-gas-benchmarks.outputs.repetitions }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: ${{ fromJson(needs.run-gas-benchmarks.outputs.matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Setup SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.PRIVKEY }} | |
| - name: Run genesis init speed benchmarks | |
| run: | | |
| CLIENT=${{ matrix.client }} | |
| SIZE=${{ matrix.size }} | |
| REPETITIONS=${{ needs.run-gas-benchmarks.outputs.repetitions }} | |
| ROOT_DIR=${{ needs.run-gas-benchmarks.outputs.root_dir }} | |
| DATE=${{ needs.run-gas-benchmarks.outputs.date }} | |
| ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF | |
| set +e | |
| echo "=== Run genesis init speed benchmarks for client: $CLIENT with size: $SIZE and date: $DATE ===" | |
| cd "$ROOT_DIR/genesis-init-benchmarks" | |
| git pull | |
| cp ../benchmarks-data-result/images.yaml . | |
| ./runSpeed.sh -t "tests/" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE/speed" -s "$SIZE" | |
| python3 report_metadata.py "$DATE" speed | |
| cp "results-$DATE/speed/reports/speed.html" "../benchmarks-data-result/genesis-data/$DATE.speed.html" | |
| cp "results-$DATE/speed/reports/speed.json" "../benchmarks-data-result/genesis-data/$DATE.speed.json" | |
| cp "results-$DATE/speed/reports/metadata.json" "../benchmarks-data-result/genesis-data/$DATE.metadata.json" | |
| set -e | |
| EOF | |
| run-genesis-init-memory-benchmarks: | |
| runs-on: ubuntu-latest | |
| needs: run-genesis-init-speed-benchmarks | |
| outputs: | |
| matrix: ${{ needs.run-genesis-init-speed-benchmarks.outputs.matrix }} | |
| date: ${{ needs.run-genesis-init-speed-benchmarks.outputs.date }} | |
| root_dir: ${{ needs.run-genesis-init-speed-benchmarks.outputs.root_dir }} | |
| clients: ${{ needs.run-genesis-init-speed-benchmarks.outputs.clients }} | |
| sizes: ${{ needs.run-genesis-init-speed-benchmarks.outputs.sizes }} | |
| repetitions: ${{ needs.run-genesis-init-speed-benchmarks.outputs.repetitions }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: ${{ fromJson(needs.run-genesis-init-speed-benchmarks.outputs.matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Setup SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.PRIVKEY }} | |
| - name: Run genesis init memory benchmarks | |
| run: | | |
| CLIENT=${{ matrix.client }} | |
| SIZE=${{ matrix.size }} | |
| REPETITIONS=${{ needs.run-genesis-init-speed-benchmarks.outputs.repetitions }} | |
| ROOT_DIR=${{ needs.run-genesis-init-speed-benchmarks.outputs.root_dir }} | |
| DATE=${{ needs.run-genesis-init-speed-benchmarks.outputs.date }} | |
| ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF | |
| set +e | |
| echo "=== Run genesis init memory benchmarks for client: $CLIENT with size: $SIZE and date: $DATE ===" | |
| cd "$ROOT_DIR/genesis-init-benchmarks" | |
| git pull | |
| cp ../benchmarks-data-result/images.yaml . | |
| ./runMemory.sh -t "tests/" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE/memory" -s "$SIZE" | |
| python3 report_metadata.py "$DATE" memory | |
| cp "results-$DATE/memory/reports/memory.html" "../benchmarks-data-result/genesis-data/$DATE.memory.html" | |
| cp "results-$DATE/memory/reports/memory.json" "../benchmarks-data-result/genesis-data/$DATE.memory.json" | |
| cp "results-$DATE/memory/reports/metadata.json" "../benchmarks-data-result/genesis-data/$DATE.metadata.json" | |
| python3 merge_result.py "$DATE" | |
| cp "results-$DATE/result.json" "../benchmarks-data-result/genesis-data/$DATE.json" | |
| set -e | |
| EOF | |
| run-burntpix-benchmarks: | |
| runs-on: ubuntu-latest | |
| needs: run-genesis-init-memory-benchmarks | |
| outputs: | |
| matrix: ${{ needs.run-genesis-init-memory-benchmarks.outputs.matrix }} | |
| date: ${{ needs.run-genesis-init-memory-benchmarks.outputs.date }} | |
| root_dir: ${{ needs.run-genesis-init-memory-benchmarks.outputs.root_dir }} | |
| clients: ${{ needs.run-genesis-init-memory-benchmarks.outputs.clients }} | |
| sizes: ${{ needs.run-genesis-init-memory-benchmarks.outputs.sizes }} | |
| repetitions: ${{ needs.run-genesis-init-memory-benchmarks.outputs.repetitions }} | |
| steps: | |
| - name: Setup SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.PRIVKEY }} | |
| - name: Run burntpix benchmarks | |
| run: | | |
| # Use const due to can't handle ${{ needs.run-genesis-init-memory-benchmarks.outputs.clients }} | |
| CLIENT="nethermind,geth,reth,erigon,besu" | |
| REPETITIONS=${{ needs.run-genesis-init-memory-benchmarks.outputs.repetitions }} | |
| ROOT_DIR=${{ needs.run-genesis-init-memory-benchmarks.outputs.root_dir }} | |
| DATE=${{ needs.run-genesis-init-memory-benchmarks.outputs.date }} | |
| ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF | |
| set +e | |
| echo "=== Run burntpix benchmarks for client: $CLIENT with date: $DATE ===" | |
| cd "$ROOT_DIR/burntpix-benchmarks" | |
| git pull | |
| cp ../benchmarks-data-result/images.yaml . | |
| ./run.sh -t "testburnt/" -w "./testburntwarm/warm.txt" -c "$CLIENT" -r "$REPETITIONS" -o "results-$DATE" | |
| python3 report_metadata.py "$DATE" | |
| cp "results-$DATE/reports/index.html" "../benchmarks-data-result/burntpix/$DATE.html" | |
| cp "results-$DATE/reports/result.json" "../benchmarks-data-result/burntpix/$DATE.json" | |
| cp "results-$DATE/reports/metadata.json" "../benchmarks-data-result/burntpix/$DATE.metadata.json" | |
| set -e | |
| EOF | |
| push-results: | |
| runs-on: ubuntu-latest | |
| needs: run-burntpix-benchmarks | |
| steps: | |
| - name: Setup SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.PRIVKEY }} | |
| - name: Push results | |
| run: | | |
| ROOT_DIR=${{ needs.run-burntpix-benchmarks.outputs.root_dir }} | |
| ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 root@${{ secrets.REMOTE_IP }} << EOF | |
| set +e | |
| cd "$ROOT_DIR/benchmarks-data-result" | |
| git status | |
| ./push.sh | |
| set -e | |
| EOF |