Add test for INC examples

.github/workflows/test_inc_examples.yml

@@ -0,0 +1,47 @@
+name: INC - Examples Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: 0 1 * * 1  # run weekly: every Monday at 1am
+  push:
+    paths:
+    - '.github/workflows/test_inc_examples.yml'
+    - 'examples/neural_compressor/*'
+  pull_request:
+    paths:
+    - '.github/workflows/test_inc_examples.yml'
+    - 'examples/neural_compressor/*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.10"]
+
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        pip install optimum[neural-compressor,ipex] pytest
+        pip install -r examples/neural_compressor/text-classification/requirements.txt
+        pip install -r examples/neural_compressor/question-answering/requirements.txt
+        pip install -r examples/neural_compressor/token-classification/requirements.txt
+        pip install -r examples/neural_compressor/language-modeling/requirements.txt
+        pip install -r examples/neural_compressor/multiple-choice/requirements.txt
+
+    - name: Test examples
+      run: |
+        python -m pytest examples/neural_compressor/test_examples.py

examples/neural_compressor/test_examples.py

@@ -59,7 +59,7 @@ def test_run_glue(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
             test_args = f"""
                 run_glue.py
-                --model_name_or_path distilbert-base-uncased-finetuned-sst-2-english
+                --model_name_or_path hf-internal-testing/tiny-random-DistilBertForSequenceClassification
                 --task_name sst2
                 --apply_quantization
                 --apply_pruning
@@ -79,13 +79,12 @@ def test_run_glue(self):
             with patch.object(sys, "argv", test_args):
                 run_glue.main()
                 results = get_results(tmp_dir)
-                self.assertGreaterEqual(results["eval_accuracy"], 0.70)
 
     def test_run_qa(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
             test_args = f"""
                 run_qa.py
-                --model_name_or_path distilbert-base-uncased-distilled-squad
+                --model_name_or_path hf-internal-testing/tiny-random-DistilBertForQuestionAnswering
                 --dataset_name squad
                 --apply_quantization
                 --apply_pruning
@@ -105,14 +104,12 @@ def test_run_qa(self):
             with patch.object(sys, "argv", test_args):
                 run_qa.main()
                 results = get_results(tmp_dir)
-                self.assertGreaterEqual(results["eval_f1"], 70)
-                self.assertGreaterEqual(results["eval_exact_match"], 70)
 
     def test_run_ner(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
             test_args = f"""
                 run_ner.py
-                --model_name_or_path elastic/distilbert-base-uncased-finetuned-conll03-english
+                --model_name_or_path hf-internal-testing/tiny-random-RobertaForTokenClassification
                 --dataset_name conll2003
                 --apply_quantization
                 --apply_pruning
@@ -132,16 +129,12 @@ def test_run_ner(self):
             with patch.object(sys, "argv", test_args):
                 run_ner.main()
                 results = get_results(tmp_dir)
-                self.assertGreaterEqual(results["eval_accuracy"], 0.70)
-                self.assertGreaterEqual(results["eval_f1"], 0.70)
-                self.assertGreaterEqual(results["eval_precision"], 0.70)
-                self.assertGreaterEqual(results["eval_recall"], 0.70)
 
     def test_run_swag(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
             test_args = f"""
                 run_swag.py
-                --model_name_or_path ehdwns1516/bert-base-uncased_SWAG
+                --model_name_or_path hf-internal-testing/tiny-random-AlbertForMultipleChoice
                 --apply_quantization
                 --apply_pruning
                 --target_sparsity 0.02
@@ -160,15 +153,14 @@ def test_run_swag(self):
             with patch.object(sys, "argv", test_args):
                 run_swag.main()
                 results = get_results(tmp_dir)
-                self.assertGreaterEqual(results["eval_accuracy"], 0.60)
 
     def test_run_clm(self):
         quantization_approach = "dynamic"
 
         with tempfile.TemporaryDirectory() as tmp_dir:
             test_args = f"""
                 run_clm.py
-                --model_name_or_path EleutherAI/gpt-neo-125M
+                --model_name_or_path hf-internal-testing/tiny-random-GPT2LMHeadModel
                 --dataset_name wikitext
                 --dataset_config_name wikitext-2-raw-v1
                 --apply_quantization
@@ -190,15 +182,14 @@ def test_run_clm(self):
             with patch.object(sys, "argv", test_args):
                 run_clm.main()
                 results = get_results(tmp_dir)
-                self.assertLessEqual(results["eval_loss"], 15)
 
     def test_run_mlm(self):
         quantization_approach = "static"
 
         with tempfile.TemporaryDirectory() as tmp_dir:
             test_args = f"""
                 run_mlm.py
-                --model_name_or_path google/electra-small-discriminator
+                --model_name_or_path hf-internal-testing/tiny-random-DistilBertForMaskedLM
                 --dataset_name wikitext
                 --dataset_config_name wikitext-2-raw-v1
                 --apply_quantization
@@ -220,7 +211,6 @@ def test_run_mlm(self):
             with patch.object(sys, "argv", test_args):
                 run_mlm.main()
                 results = get_results(tmp_dir)
-                self.assertLessEqual(results["eval_loss"], 15)
 
 
 if __name__ == "__main__":