-
-
Notifications
You must be signed in to change notification settings - Fork 3.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Compare screenshots with main on PRs (#13248)
# Objective - Compare screenshots for a few examples between PRs and main ## Solution - Send screenshots taken to a screenshot comparison service - Not completely sure every thing will work at once, but it shouldn't break anything at least - it needs a secret to work, I'll add it if enough people agree with this PR - this PR doesn't change anything on the screenshot selection (load_gltf and breakout currently), this will need rendering folks input and can happen later --------- Co-authored-by: Alice Cecile <[email protected]>
- Loading branch information
1 parent
89e19aa
commit 82d6f56
Showing
4 changed files
with
184 additions
and
47 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| name: CI - examples | ||
|
|
||
| on: | ||
| merge_group: | ||
| pull_request_target: | ||
| push: | ||
| branches: | ||
| - main | ||
| - release-* | ||
|
|
||
| # Unlike jobs in ci.yml, jobs in this workflow can access secrets. Their definitions are taken from the base branch of a PR, not from the PR itself. | ||
|
|
||
| env: | ||
| CARGO_TERM_COLOR: always | ||
| # If nightly is breaking CI, modify this variable to target a specific nightly version. | ||
| NIGHTLY_TOOLCHAIN: nightly | ||
|
|
||
| concurrency: | ||
| group: ${{github.workflow}}-${{github.ref}}-examples | ||
| cancel-in-progress: ${{github.event_name == 'pull_request_target'}} | ||
|
|
||
| jobs: | ||
| run-examples-macos-metal: | ||
| # Explicity use macOS 14 to take advantage of M1 chip. | ||
| runs-on: macos-14 | ||
| timeout-minutes: 30 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: dtolnay/rust-toolchain@stable | ||
| - name: Disable audio | ||
| # Disable audio through a patch. on github m1 runners, audio timeouts after 15 minutes | ||
| run: git apply --ignore-whitespace tools/example-showcase/disable-audio.patch | ||
| - name: Build bevy | ||
| # this uses the same command as when running the example to ensure build is reused | ||
| run: | | ||
| TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome" | ||
| - name: Run examples | ||
| run: | | ||
| for example in .github/example-run/*.ron; do | ||
| example_name=`basename $example .ron` | ||
| echo -n $example_name > last_example_run | ||
| echo "running $example_name - "`date` | ||
| time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome" | ||
| sleep 10 | ||
| if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then | ||
| mkdir screenshots-$example_name | ||
| mv screenshot-*.png screenshots-$example_name/ | ||
| fi | ||
| done | ||
| mkdir traces && mv trace*.json traces/ | ||
| mkdir screenshots && mv screenshots-* screenshots/ | ||
| - name: save traces | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: example-traces-macos | ||
| path: traces | ||
| - name: save screenshots | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: screenshots-macos | ||
| path: screenshots | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ failure() && github.event_name == 'pull_request' }} | ||
| with: | ||
| name: example-run-macos | ||
| path: example-run/ | ||
|
|
||
| compare-macos-screenshots: | ||
| name: Compare macOS screenshots | ||
| needs: [run-examples-macos-metal] | ||
| uses: ./.github/workflows/send-screenshots-to-pixeleagle.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| artifact: screenshots-macos | ||
| os: macos | ||
| secrets: inherit |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| name: Send Screenshots to Pixel Eagle | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| artifact: | ||
| required: true | ||
| type: string | ||
| commit: | ||
| required: true | ||
| type: string | ||
| branch: | ||
| required: true | ||
| type: string | ||
| os: | ||
| required: true | ||
| type: string | ||
|
|
||
| jobs: | ||
| send-to-pixel-eagle: | ||
| name: Send screenshots to Pixel Eagle | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
|
|
||
| - name: Download artifact | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| pattern: ${{ inputs.artifact }} | ||
|
|
||
| - name: Send to Pixel Eagle | ||
| env: | ||
| project: B04F67C0-C054-4A6F-92EC-F599FEC2FD1D | ||
| run: | | ||
| # Create a new run with its associated metadata | ||
| metadata='{"os":"${{ inputs.os }}", "commit": "${{ inputs.commit }}", "branch": "${{ inputs.branch }}"}' | ||
| run=`curl https://pixel-eagle.vleue.com/$project/runs --json "$metadata" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.id'` | ||
| SAVEIFS=$IFS | ||
| cd ${{ inputs.artifact }} | ||
| # Read the hashes of the screenshot for fast comparison when they are equal | ||
| IFS=$'\n' | ||
| # Build a json array of screenshots and their hashes | ||
| hashes='['; | ||
| for screenshot in $(find . -type f -name "*.png"); | ||
| do | ||
| name=${screenshot:14} | ||
| echo $name | ||
| hash=`shasum -a 256 $screenshot | awk '{print $1}'` | ||
| hashes="$hashes [\"$name\",\"$hash\"]," | ||
| done | ||
| hashes=`echo $hashes | rev | cut -c 2- | rev` | ||
| hashes="$hashes]" | ||
| IFS=$SAVEIFS | ||
| # Upload screenshots with unknown hashes | ||
| curl https://pixel-eagle.vleue.com/$project/runs/$run/hashes --json "$hashes" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.[]|[.name] | @tsv' | | ||
| while IFS=$'\t' read -r name; do | ||
| name=`echo $name | tr -d '"'` | ||
| echo "Uploading $name" | ||
| curl https://pixel-eagle.vleue.com/$project/runs/$run/screenshots -F "data=@./$name" -F "screenshot=$name" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | ||
| echo | ||
| done | ||
| IFS=$SAVEIFS | ||
| cd .. | ||
| # Trigger comparison with the main branch on the same os | ||
| curl https://pixel-eagle.vleue.com/$project/runs/$run/compare/auto --json '{"os":"<equal>", "branch": "main"}' --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} > pixeleagle.json | ||
| # Log results | ||
| compared_with=`cat pixeleagle.json | jq '.to'` | ||
| status=0 | ||
| missing=`cat pixeleagle.json | jq '.missing | length'` | ||
| if [ ! $missing -eq 0 ]; then | ||
| echo "There are $missing missing screenshots" | ||
| echo "::warning title=$missing missing screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to" | ||
| status=1 | ||
| fi | ||
| diff=`cat pixeleagle.json | jq '.diff | length'` | ||
| if [ ! $diff -eq 0 ]; then | ||
| echo "There are $diff screenshots with a difference" | ||
| echo "::warning title=$diff different screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to" | ||
| status=1 | ||
| fi | ||
| echo "created run $run: https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to" | ||
| exit $status |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters