-
Notifications
You must be signed in to change notification settings - Fork 27.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'huggingface:main' into add_diffllama
- Loading branch information
Showing
864 changed files
with
66,400 additions
and
23,548 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,43 +1,74 @@ | ||
| name: Self-hosted runner (benchmark) | ||
|
|
||
| on: | ||
| schedule: | ||
| - cron: "17 2 * * *" | ||
| workflow_call: | ||
| push: | ||
| branches: [main] | ||
| pull_request: | ||
| types: [ opened, labeled, reopened, synchronize ] | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | ||
| cancel-in-progress: true | ||
|
|
||
| env: | ||
| HF_HOME: /mnt/cache | ||
| TF_FORCE_GPU_ALLOW_GROWTH: true | ||
|
|
||
|
|
||
| jobs: | ||
| benchmark: | ||
| name: Benchmark | ||
| runs-on: | ||
| group: aws-g5-4xlarge-cache | ||
| strategy: | ||
| matrix: | ||
| group: [aws-g5-4xlarge-cache, aws-p4d-24xlarge-plus] | ||
| runs-on: | ||
| group: ${{ matrix.group }} | ||
| if: | | ||
| (github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'run-benchmark') )|| | ||
| (github.event_name == 'push' && github.ref == 'refs/heads/main') | ||
| container: | ||
| image: huggingface/transformers-all-latest-gpu | ||
| options: --gpus all --privileged --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ | ||
| image: huggingface/transformers-pytorch-gpu | ||
| options: --gpus all --privileged --ipc host | ||
| steps: | ||
| - name: Update clone | ||
| working-directory: /transformers | ||
| - name: Get repo | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | ||
|
|
||
| - name: Install libpq-dev & psql | ||
| run: | | ||
| git fetch && git checkout ${{ github.sha }} | ||
| apt update | ||
| apt install -y libpq-dev postgresql-client | ||
| - name: Install benchmark script dependencies | ||
| run: python3 -m pip install -r benchmark/requirements.txt | ||
|
|
||
| - name: Reinstall transformers in edit mode (remove the one installed during docker image build) | ||
| working-directory: /transformers | ||
| run: python3 -m pip uninstall -y transformers && python3 -m pip install -e . | ||
| run: python3 -m pip uninstall -y transformers && python3 -m pip install -e ".[torch]" | ||
|
|
||
| - name: Benchmark (daily) | ||
| if: github.event_name == 'schedule' | ||
| working-directory: /transformers | ||
| - name: Run database init script | ||
| run: | | ||
| python3 -m pip install optimum-benchmark>=0.3.0 | ||
| HF_TOKEN=${{ secrets.TRANSFORMERS_BENCHMARK_TOKEN }} python3 benchmark/benchmark.py --repo_id hf-internal-testing/benchmark_results --path_in_repo $(date +'%Y-%m-%d') --config-dir benchmark/config --config-name generation --commit=${{ github.sha }} backend.model=google/gemma-2b backend.cache_implementation=null,static backend.torch_compile=false,true --multirun | ||
| psql -f benchmark/init_db.sql | ||
| env: | ||
| PGDATABASE: metrics | ||
| PGHOST: ${{ secrets.TRANSFORMERS_BENCHMARKS_PGHOST }} | ||
| PGUSER: transformers_benchmarks | ||
| PGPASSWORD: ${{ secrets.TRANSFORMERS_BENCHMARKS_PGPASSWORD }} | ||
|
|
||
| - name: Benchmark (merged to main event) | ||
| if: github.event_name == 'push' && github.ref_name == 'main' | ||
| working-directory: /transformers | ||
| - name: Run benchmark | ||
| run: | | ||
| python3 -m pip install optimum-benchmark>=0.3.0 | ||
| HF_TOKEN=${{ secrets.TRANSFORMERS_BENCHMARK_TOKEN }} python3 benchmark/benchmark.py --repo_id hf-internal-testing/benchmark_results_merge_event --path_in_repo $(date +'%Y-%m-%d') --config-dir benchmark/config --config-name generation --commit=${{ github.sha }} backend.model=google/gemma-2b backend.cache_implementation=null,static backend.torch_compile=false,true --multirun | ||
| git config --global --add safe.directory /__w/transformers/transformers | ||
| if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then | ||
| commit_id=$(echo "${{ github.event.pull_request.head.sha }}") | ||
| elif [ "$GITHUB_EVENT_NAME" = "push" ]; then | ||
| commit_id=$GITHUB_SHA | ||
| fi | ||
| commit_msg=$(git show -s --format=%s | cut -c1-70) | ||
| python3 benchmark/llama.py "${{ github.head_ref || github.ref_name }}" "$commit_id" "$commit_msg" | ||
| env: | ||
| HF_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} | ||
| # Enable this to see debug logs | ||
| # HF_HUB_VERBOSITY: debug | ||
| # TRANSFORMERS_VERBOSITY: debug | ||
| PGHOST: ${{ secrets.TRANSFORMERS_BENCHMARKS_PGHOST }} | ||
| PGUSER: transformers_benchmarks | ||
| PGPASSWORD: ${{ secrets.TRANSFORMERS_BENCHMARKS_PGPASSWORD }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| name: Process failed tests | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| docker: | ||
| required: true | ||
| type: string | ||
| start_sha: | ||
| required: true | ||
| type: string | ||
|
|
||
|
|
||
| env: | ||
| HF_HOME: /mnt/cache | ||
| TRANSFORMERS_IS_CI: yes | ||
| OMP_NUM_THREADS: 8 | ||
| MKL_NUM_THREADS: 8 | ||
| RUN_SLOW: yes | ||
| # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access. | ||
| # This token is created under the bot `hf-transformers-bot`. | ||
| HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} | ||
| SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }} | ||
| TF_FORCE_GPU_ALLOW_GROWTH: true | ||
| RUN_PT_TF_CROSS_TESTS: 1 | ||
| CUDA_VISIBLE_DEVICES: 0,1 | ||
|
|
||
|
|
||
| jobs: | ||
| run_models_gpu: | ||
| name: " " | ||
| runs-on: | ||
| group: aws-g4dn-2xlarge-cache | ||
| container: | ||
| image: ${{ inputs.docker }} | ||
| options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ | ||
| steps: | ||
| - uses: actions/download-artifact@v4 | ||
| with: | ||
| name: ci_results_run_models_gpu | ||
| path: /transformers/ci_results_run_models_gpu | ||
|
|
||
| - name: Update clone | ||
| working-directory: /transformers | ||
| run: git fetch && git checkout ${{ github.sha }} | ||
|
|
||
| - name: Get target commit | ||
| working-directory: /transformers/utils | ||
| run: | | ||
| echo "END_SHA=$(TOKEN=${{ secrets.ACCESS_REPO_INFO_TOKEN }} python3 -c 'import os; from get_previous_daily_ci import get_last_daily_ci_run_commit; commit=get_last_daily_ci_run_commit(token=os.environ["TOKEN"]); print(commit)')" >> $GITHUB_ENV | ||
| - name: Checkout to `start_sha` | ||
| working-directory: /transformers | ||
| run: git fetch && git checkout ${{ inputs.start_sha }} | ||
|
|
||
| - name: Reinstall transformers in edit mode (remove the one installed during docker image build) | ||
| working-directory: /transformers | ||
| run: python3 -m pip uninstall -y transformers && python3 -m pip install -e . | ||
|
|
||
| - name: NVIDIA-SMI | ||
| run: | | ||
| nvidia-smi | ||
| - name: Environment | ||
| working-directory: /transformers | ||
| run: | | ||
| python3 utils/print_env.py | ||
| - name: Show installed libraries and their versions | ||
| working-directory: /transformers | ||
| run: pip freeze | ||
|
|
||
| - name: Check failed tests | ||
| working-directory: /transformers | ||
| run: python3 utils/check_bad_commit.py --start_commit ${{ inputs.start_sha }} --end_commit ${{ env.END_SHA }} --file ci_results_run_models_gpu/new_model_failures.json --output_file new_model_failures_with_bad_commit.json | ||
|
|
||
| - name: Show results | ||
| working-directory: /transformers | ||
| run: | | ||
| ls -l new_model_failures_with_bad_commit.json | ||
| cat new_model_failures_with_bad_commit.json | ||
| - name: Checkout back | ||
| working-directory: /transformers | ||
| run: | | ||
| git checkout ${{ inputs.start_sha }} | ||
| - name: Process report | ||
| shell: bash | ||
| working-directory: /transformers | ||
| env: | ||
| TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }} | ||
| run: | | ||
| python3 utils/process_bad_commit_report.py | ||
| - name: Process report | ||
| shell: bash | ||
| working-directory: /transformers | ||
| env: | ||
| TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }} | ||
| run: | | ||
| { | ||
| echo 'REPORT_TEXT<<EOF' | ||
| python3 utils/process_bad_commit_report.py | ||
| echo EOF | ||
| } >> "$GITHUB_ENV" | ||
| - name: Send processed report | ||
| if: ${{ !endsWith(env.REPORT_TEXT, '{}') }} | ||
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 | ||
| with: | ||
| # Slack channel id, channel name, or user id to post message. | ||
| # See also: https://api.slack.com/methods/chat.postMessage#channels | ||
| channel-id: '#transformers-ci-feedback-tests' | ||
| # For posting a rich message using Block Kit | ||
| payload: | | ||
| { | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "${{ env.REPORT_TEXT }}" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| env: | ||
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.