Merge branch 'main' into fix-ort-ci

.github/workflows/test_bettertransformer.yml

@@ -2,9 +2,9 @@ name: BetterTransformer / Python - Test
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -17,29 +17,35 @@ jobs:
       matrix:
         python-version: [3.8, 3.9]
         os: [ubuntu-20.04, macos-13]
+        exclude: [{ python-version: 3.8, os: macos-13 }]
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        pip install .[tests]
-        pip install --no-cache-dir --upgrade torch torchvision torchaudio
-        pip install accelerate
-    - name: Test on pytorch stable
-      working-directory: tests
-      run: |
-        pytest bettertransformer/test_*.py -s -vvvvv
-    - name: Install dependencies 2
-      run: |
-        pip uninstall -y torch torchvision torchaudio
-        pip install --no-cache-dir --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
-    - name: Test on pytorch nightly
-      working-directory: tests
-      run: |
-        pytest bettertransformer/test_*.py -s -vvvvv
+      - name: Checkout code
+        uses: actions/checkout@v4
 
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install .[tests]
+          pip install --no-cache-dir --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+          pip install accelerate
+
+      - name: Test with stable pytorch
+        working-directory: tests
+        run: |
+          pytest bettertransformer -s -vvvvv
+
+      - name: Install dependencies 2
+        run: |
+          pip uninstall -y torch torchvision torchaudio
+          pip install --no-cache-dir --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
+
+      - name: Test with nightly pytorch
+        working-directory: tests
+        run: |
+          pytest bettertransformer -s -vvvvv

.github/workflows/test_fx.yml

@@ -2,9 +2,9 @@ name: FX / Python - Test
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -20,16 +20,19 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        pip install .[tests]
-        pip install git+https://github.com/huggingface/transformers.git
-    - name: Test with unittest
-      working-directory: tests
-      run: |
-        python -m pytest fx/optimization/test_transformations.py --exitfirst
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install .[tests]
+
+      - name: Test with pytest
+        working-directory: tests
+        run: |
+          python -m pytest -s -v -x fx/optimization

.github/workflows/test_utils.yml

@@ -0,0 +1,40 @@
+name: Utils / Python - Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-13]
+        python-version: [3.8, 3.9]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+          pip install .[tests]
+
+      - name: Test with pytest
+        working-directory: tests
+        run: |
+          python -m pytest -s -vvvv utils

docs/source/exporters/onnx/overview.mdx

@@ -77,6 +77,7 @@ Supported architectures from [🤗 Transformers](https://huggingface.co/docs/tra
 - Pegasus
 - Perceiver
 - Phi
+- Phi3
 - Pix2Struct
 - PoolFormer
 - Qwen2(Qwen1.5)

optimum/exporters/onnx/model_configs.py

@@ -51,6 +51,7 @@
     NormalizedSeq2SeqConfig,
     NormalizedTextAndVisionConfig,
     NormalizedTextConfig,
+    NormalizedTextConfigWithGQA,
     NormalizedVisionConfig,
     is_diffusers_available,
     logging,
@@ -291,6 +292,14 @@ class PhiOnnxConfig(TextDecoderWithPositionIdsOnnxConfig):
     NORMALIZED_CONFIG_CLASS = NormalizedTextConfig
 
 
+class Phi3OnnxConfig(PhiOnnxConfig):
+    DUMMY_INPUT_GENERATOR_CLASSES = (
+        MistralDummyPastKeyValuesGenerator,
+    ) + TextDecoderOnnxConfig.DUMMY_INPUT_GENERATOR_CLASSES
+    DUMMY_PKV_GENERATOR_CLASS = MistralDummyPastKeyValuesGenerator
+    NORMALIZED_CONFIG_CLASS = NormalizedTextConfigWithGQA
+
+
 class MistralOnnxConfig(TextDecoderWithPositionIdsOnnxConfig):
     # This is because of the patching of torch.triu in AttentionMaskConverter, that exists from transformers>=4.35
     MIN_TRANSFORMERS_VERSION = version.parse("4.34.99")

optimum/exporters/onnx/utils.py

@@ -84,6 +84,7 @@
     "llama",
     "mistral",
     "phi",
+    "phi3",
     "qwen2",
 }
 

optimum/exporters/tasks.py

@@ -893,6 +893,14 @@ class TasksManager:
             "text-classification",
             onnx="PhiOnnxConfig",
         ),
+        "phi3": supported_tasks_mapping(
+            "feature-extraction",
+            "feature-extraction-with-past",
+            "text-generation",
+            "text-generation-with-past",
+            "text-classification",
+            onnx="Phi3OnnxConfig",
+        ),
         "pix2struct": supported_tasks_mapping(
             "image-to-text",
             "image-to-text-with-past",

optimum/fx/optimization/transformations.py

@@ -19,15 +19,31 @@
 import operator
 import warnings
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, List
+from typing import List
 
 import torch
+from torch.fx import GraphModule, Node
 from transformers.file_utils import add_end_docstrings
-from transformers.utils.fx import _gen_constructor_wrapper
 
 
-if TYPE_CHECKING:
-    from torch.fx import GraphModule, Node
+try:
+    from transformers.utils.fx import _gen_constructor_wrapper
+except ImportError:
+    from transformers.utils.fx import gen_constructor_wrapper
+
+    def _gen_constructor_wrapper(*args, **kwargs):
+        wrapper, target = gen_constructor_wrapper(*args, **kwargs)
+
+        def wrapper_with_forced_tracing(*_args, **_kwargs):
+            import torch.fx._symbolic_trace
+
+            orginal_flag = torch.fx._symbolic_trace._is_fx_tracing_flag
+            torch.fx._symbolic_trace._is_fx_tracing_flag = True
+            out = wrapper(*_args, **_kwargs)
+            torch.fx._symbolic_trace._is_fx_tracing_flag = orginal_flag
+            return out
+
+        return wrapper_with_forced_tracing, target
 
 
 _ATTRIBUTES_DOCSTRING = r"""

optimum/pipelines/pipelines_base.py

@@ -45,7 +45,7 @@
 from transformers.pipelines import infer_framework_load_model
 
 from ..bettertransformer import BetterTransformer
-from ..utils import is_onnxruntime_available
+from ..utils import check_if_transformers_greater, is_onnxruntime_available
 from ..utils.file_utils import find_files_matching_pattern
 
 
@@ -179,7 +179,12 @@ def load_bettertransformer(
     **kwargs,
 ):
     if model_kwargs is None:
-        model_kwargs = {}
+        # the argument was first introduced in 4.36.0 but most models didn't have an sdpa implementation then
+        # see https://github.com/huggingface/transformers/blob/v4.36.0/src/transformers/modeling_utils.py#L1258
+        if check_if_transformers_greater("4.36.0"):
+            model_kwargs = {"attn_implementation": "eager"}
+        else:
+            model_kwargs = {}
 
     if model is None:
         model_id = SUPPORTED_TASKS[targeted_task]["default"]

optimum/utils/__init__.py

@@ -80,5 +80,6 @@
     NormalizedSeq2SeqConfig,
     NormalizedTextAndVisionConfig,
     NormalizedTextConfig,
+    NormalizedTextConfigWithGQA,
     NormalizedVisionConfig,
 )

optimum/utils/preprocessing/task_processors_manager.py

@@ -23,7 +23,7 @@
 
 
 if TYPE_CHECKING:
-    from .base import DatasetProcessing
+    from .base import TaskProcessor
 
 
 class TaskProcessorsManager:
@@ -35,7 +35,7 @@ class TaskProcessorsManager:
     }
 
     @classmethod
-    def get_task_processor_class_for_task(cls, task: str) -> Type:
+    def get_task_processor_class_for_task(cls, task: str) -> Type["TaskProcessor"]:
         if task not in cls._TASK_TO_DATASET_PROCESSING_CLASS:
             supported_tasks = ", ".join(cls._TASK_TO_DATASET_PROCESSING_CLASS.keys())
             raise KeyError(
@@ -45,5 +45,5 @@ def get_task_processor_class_for_task(cls, task: str) -> Type:
         return cls._TASK_TO_DATASET_PROCESSING_CLASS[task]
 
     @classmethod
-    def for_task(cls, task: str, *dataset_processing_args, **dataset_processing_kwargs: Any) -> "DatasetProcessing":
+    def for_task(cls, task: str, *dataset_processing_args, **dataset_processing_kwargs: Any) -> "TaskProcessor":
         return cls.get_task_processor_class_for_task(task)(*dataset_processing_args, **dataset_processing_kwargs)

setup.py

@@ -15,7 +15,7 @@
 REQUIRED_PKGS = [
     "coloredlogs",
     "sympy",
-    "transformers[sentencepiece]>=4.26.0,<4.41.0",
+    "transformers[sentencepiece]>=4.26.0,<4.42.0",
     "torch>=1.11",
     "packaging",
     "numpy",

tests/bettertransformer/test_encoder.py

@@ -114,7 +114,7 @@ def test_inference_speed(self):
         """
         model_name = "bert-base-uncased"
 
-        hf_model = AutoModel.from_pretrained(model_name).eval()
+        hf_model = AutoModel.from_pretrained(model_name, attn_implementation="eager").eval()
         bt_model = BetterTransformer.transform(hf_model, keep_original_model=True)
 
         BATCH_SIZE = 8

tests/bettertransformer/testing_utils.py

@@ -235,7 +235,7 @@ def _test_logits(self, model_id: str, model_type: str, **preprocessor_kwargs):
         inputs = self.prepare_inputs_for_class(model_id=model_id, model_type=model_type, **preprocessor_kwargs)
 
         torch.manual_seed(0)
-        hf_random_model = AutoModel.from_pretrained(model_id).eval()
+        hf_random_model = AutoModel.from_pretrained(model_id, attn_implementation="eager").eval()
         random_config = hf_random_model.config
 
         hf_random_model = hf_random_model.eval()

tests/exporters/exporters_utils.py

@@ -132,6 +132,7 @@
         "hf-internal-testing/tiny-random-vision_perceiver_conv": ["image-classification"],
     },
     "phi": "echarlaix/tiny-random-PhiForCausalLM",
+    "phi3": "Xenova/tiny-random-Phi3ForCausalLM",
     "pix2struct": "fxmarty/pix2struct-tiny-random",
     # "rembert": "google/rembert",
     "poolformer": "hf-internal-testing/tiny-random-PoolFormerModel",

tests/fx/optimization/test_transformations.py

@@ -86,7 +86,8 @@ def transform(self, graph_module):
 
 
 def get_bert_model():
-    model = BertModel.from_pretrained(_MODEL_NAME)
+    # sdpa attn became default
+    model = BertModel.from_pretrained(_MODEL_NAME, attn_implementation="eager")
     model.eval()
     traced = symbolic_trace(model, input_names=["input_ids", "attention_mask", "token_type_ids"])
     return model, traced

tests/onnxruntime/test_modeling.py

@@ -2258,6 +2258,7 @@ class ORTModelForCausalLMIntegrationTest(ORTModelTestMixin):
         "llama",
         "mistral",
         "mpt",
+        "phi3",
         "qwen2",
     ]
 

tests/onnxruntime/utils_onnxruntime_tests.py

@@ -128,6 +128,7 @@
     "pegasus": "hf-internal-testing/tiny-random-PegasusModel",
     "perceiver_text": "hf-internal-testing/tiny-random-language_perceiver",
     "perceiver_vision": "hf-internal-testing/tiny-random-vision_perceiver_conv",
+    "phi3": "Xenova/tiny-random-Phi3ForCausalLM",
     "pix2struct": "fxmarty/pix2struct-tiny-random",
     "poolformer": "hf-internal-testing/tiny-random-PoolFormerModel",
     "qwen2": "fxmarty/tiny-dummy-qwen2",