[PT FE] Report unsupported operations with model list (openvinotoolki…

…t#21607)

* [PT FE] Report unsupported operations with model list

* Continue on error

* Disable more models

* Check if file exists

* continue of error

* Fix on failure

* Fix file path

* Apply suggestions from code review

.github/workflows/job_pytorch_models_tests.yml

@@ -110,7 +110,13 @@ jobs:
           TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}}
           TEST_DEVICE: CPU
           USE_SYSTEM_CACHE: False
+          OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log
 
+      - name: Reformat unsupported ops file
+        if: '!cancelled()'
+        run: |
+          python3 ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/scripts/process_op_report.py ${INSTALL_TEST_DIR}/TEST-torch_unsupported_ops.log
+  
       - name: Available storage after tests
         run: |
           echo "Available storage:"
@@ -122,5 +128,5 @@ jobs:
         with:
           name: test-results-torch-models
           path: |
-            ${{ env.INSTALL_TEST_DIR }}/TEST*.html
+            ${{ env.INSTALL_TEST_DIR }}/TEST-torch*
           if-no-files-found: 'error'

tests/model_hub_tests/models_hub_common/test_convert_model.py

@@ -87,6 +87,7 @@ def teardown_method(self):
         gc.collect()
 
     def _run(self, model_name, model_link, ie_device):
+        self.model_name = model_name
         print("Load the model {} (url: {})".format(model_name, model_link))
         fw_model = self.load_model(model_name, model_link)
         print("Retrieve inputs info")

tests/model_hub_tests/torch_tests/scripts/process_op_report.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+
+if __name__ == "__main__":
+    assert len(sys.argv) > 1, "Please provide filename"
+    filename = sys.argv[1]
+    if os.path.isfile(filename):
+        ops_dict = dict()
+        with open(filename, 'r') as f:
+            for line in f.readlines():
+                r = line.split()
+                if r[0] in ops_dict:
+                    ops_dict[r[0]].append(r[1])
+                else:
+                    ops_dict[r[0]] = [r[1]]
+
+        with open(filename, 'w') as f:
+            for op in sorted(ops_dict.keys()):
+                models = ops_dict[op]
+                m_str = ', '.join(models)
+                f.write(
+                    f"{op:<30} appears in {len(models):>2} models: {m_str}\n")
+    else:
+        print(f"File {filename} doesn't exist.")

tests/model_hub_tests/torch_tests/torch_utils.py

@@ -3,6 +3,7 @@
 
 import pytest
 import torch
+import os
 from models_hub_common.test_convert_model import TestConvertModel
 from models_hub_common.utils import get_models_list
 from openvino import convert_model
@@ -27,10 +28,22 @@ def flattenize_structure(outputs):
 
 def process_pytest_marks(filepath: str):
     return [
-        pytest.param(n, marks=pytest.mark.xfail(reason=r) if m == "xfail" else pytest.mark.skip(reason=r)) if m else n
+        pytest.param(n, marks=pytest.mark.xfail(reason=r) if m ==
+                     "xfail" else pytest.mark.skip(reason=r)) if m else n
         for n, _, m, r in get_models_list(filepath)]
 
 
+def extract_unsupported_ops_from_exception(e: str) -> list:
+    exception_str = "No conversion rule found for operations:"
+    for s in e.splitlines():
+        it = s.find(exception_str)
+        if it >= 0:
+            _s = s[it + len(exception_str):]
+            ops = _s.replace(" ", "").split(",")
+            return ops
+    return []
+
+
 class TestTorchConvertModel(TestConvertModel):
     def setup_class(self):
         torch.set_grad_enabled(False)
@@ -49,8 +62,19 @@ def prepare_inputs(self, inputs_info):
             return [i.numpy() for i in inputs]
 
     def convert_model(self, model_obj):
-        ov_model = convert_model(
-            model_obj, example_input=self.example, verbose=True)
+        try:
+            ov_model = convert_model(
+                model_obj, example_input=self.example, verbose=True)
+        except Exception as e:
+            report_filename = os.environ.get("OP_REPORT_FILE", None)
+            if report_filename:
+                mode = 'a' if os.path.exists(report_filename) else 'w'
+                with open(report_filename, mode) as f:
+                    ops = extract_unsupported_ops_from_exception(str(e))
+                    if ops:
+                        ops = [f"{op} {self.model_name}" for op in ops]
+                        f.write("\n".join(ops) + "\n")
+            raise e
         return ov_model
 
     def infer_fw_model(self, model_obj, inputs):