From d1670a0da712b5194afd7e3b5211e1f9283d04cb Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 30 Apr 2024 15:31:12 +0200 Subject: [PATCH] update --- ...odel-pr-caller.yml => self-pr-slow-ci.yml} | 42 +++++++++---- ...ew_model_added.py => pr_slow_ci_models.py} | 62 +++++++++++++++++-- 2 files changed, 88 insertions(+), 16 deletions(-) rename .github/workflows/{self-new-model-pr-caller.yml => self-pr-slow-ci.yml} (73%) rename utils/{check_if_new_model_added.py => pr_slow_ci_models.py} (61%) diff --git a/.github/workflows/self-new-model-pr-caller.yml b/.github/workflows/self-pr-slow-ci.yml similarity index 73% rename from .github/workflows/self-new-model-pr-caller.yml rename to .github/workflows/self-pr-slow-ci.yml index 3bd19b6f2c3ff5..10a2156f210fbc 100644 --- a/.github/workflows/self-new-model-pr-caller.yml +++ b/.github/workflows/self-pr-slow-ci.yml @@ -4,6 +4,11 @@ on: pull_request: paths: - "src/transformers/models/*/modeling_*.py" + - "tests/models/*/test_*.py" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true env: HF_HOME: /mnt/cache @@ -20,31 +25,46 @@ env: CUDA_VISIBLE_DEVICES: 0,1 jobs: - check_for_new_model: + find_models_to_run: runs-on: ubuntu-22.04 - name: Check if a PR is a new model PR + name: Find models to run slow tests + # Triggered only if the required label `run-slow` is added + if: ${{ contains(github.event.pull_request.labels.*.name, 'run-slow') }} outputs: - new_model: ${{ steps.check_new_model.outputs.new_model }} + models: ${{ steps.models_to_run.outputs.models }} steps: - uses: actions/checkout@v4 with: fetch-depth: "0" + ref: ${{ github.event.pull_request.head.sha }} + + - name: Get commit message + run: | + echo "commit_message=$(git show -s --format=%s)" >> $GITHUB_ENV - - name: Check if there is a new model - id: check_new_model + - name: Get models to run slow tests run: | + echo "${{ env.commit_message }}" python -m pip install GitPython - echo "new_model=$(python utils/check_if_new_model_added.py | tail -n 1)" >> $GITHUB_OUTPUT + python utils/pr_slow_ci_models.py --commit_message "${{ env.commit_message }}" | tee output.txt + echo "models=$(tail -n 1 output.txt)" >> $GITHUB_ENV + + - name: Models to run slow tests + id: models_to_run + run: | + echo "${{ env.models }}" + echo "models=${{ env.models }}" >> $GITHUB_OUTPUT run_models_gpu: - name: Run all tests for the new model - # Triggered if it is a new model PR and the required label is added - if: ${{ needs.check_for_new_model.outputs.new_model != '' && contains(github.event.pull_request.labels.*.name, 'single-model-run-slow') }} - needs: check_for_new_model + name: Run all tests for the model + # Triggered only `find_models_to_run` is triggered (label `run-slow` is added) which gives the models to run + # (either a new model PR or via a commit message) + if: ${{ needs.find_models_to_run.outputs.models != '[]' }} + needs: find_models_to_run strategy: fail-fast: false matrix: - folders: ["${{ needs.check_for_new_model.outputs.new_model }}"] + folders: ${{ fromJson(needs.find_models_to_run.outputs.models) }} machine_type: [single-gpu, multi-gpu] runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, ci] container: diff --git a/utils/check_if_new_model_added.py b/utils/pr_slow_ci_models.py similarity index 61% rename from utils/check_if_new_model_added.py rename to utils/pr_slow_ci_models.py index f3ae0d585a1517..8ea20b2a9b7236 100644 --- a/utils/check_if_new_model_added.py +++ b/utils/pr_slow_ci_models.py @@ -13,18 +13,23 @@ # limitations under the License. """ -This script is used to get the directory of the modeling file that is added in a pull request (i.e. a new model PR). +This script is used to get the models for which to run slow CI. + +A new model added in a pull request will be included, as well as models specified in a commit message with a prefix +`[run-slow]`, `[run_slow]` or `[run slow]`. For example, the commit message `[run_slow]bert, gpt2` will give `bert` and +`gpt2`. Usage: ```bash -python utils/check_if_new_model_added.py +python utils/pr_slow_ci_models.py.py ``` """ +import argparse import re from pathlib import Path -from typing import List +from typing import Dict, List from git import Repo @@ -82,7 +87,7 @@ def get_new_python_files() -> List[str]: return get_new_python_files_between_commits(repo.head.commit, branching_commits) -if __name__ == "__main__": +def get_new_model(): new_files = get_new_python_files() reg = re.compile(r"src/transformers/(models/.*)/modeling_.*\.py") @@ -93,4 +98,51 @@ def get_new_python_files() -> List[str]: new_model = find_new_model[0] # It's unlikely we have 2 new modeling files in a pull request. break - print(new_model) + return new_model + + +def parse_commit_message(commit_message: str) -> str: + """ + Parses the commit message to find the models specified in it to run slow CI. + + Args: + commit_message (`str`): The commit message of the current commit. + + Returns: + `str`: The substring in `commit_message` after `[run-slow]`, [run_slow]` or [run slow]`. If no such prefix is + found, the empty string is returned. + """ + if commit_message is None: + return "" + + command_search = re.search(r"\[([^\]]*)\](.*)", commit_message) + if command_search is not None: + command = command_search.groups()[0] + command = command.lower().replace("-", " ").replace("_", " ") + run_slow = command == "run slow" + if run_slow: + models = command_search.groups()[1].strip() + return models + else: + return "" + else: + return "" + + +def get_models(commit_message: str): + models = parse_commit_message(commit_message) + return [f"models/{x}" for x in models.replace(",", " ").split()] + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument( + "--commit_message", type=str, default="", help="The commit message." + ) + args = parser.parse_args() + + new_model = get_new_model() + specified_models = get_models(args.commit_message) + models = ([] if new_model == "" else [new_model]) + specified_models + print(models)