Fix model analysis weekly pipeline failures

.github/workflows/model-analysis-weekly.yml

@@ -74,6 +74,9 @@ jobs:
           cmake --build ${{ steps.strings.outputs.build-output-dir }}
 
       - name: Run Model Analysis Script
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_HUB_DISABLE_PROGRESS_BARS: 1
         shell: bash
         run: |
           source env/activate

forge/test/models/pytorch/audio/stereo/test_stereo.py

@@ -32,4 +32,4 @@ def test_stereo(variant):
 
     compiled_model = forge.compile(framework_model, sample_inputs=inputs)
 
-    verify(inputs, framework_model, compiled_model)
+    verify(inputs, framework_model, compiled_model, module_name="pt_" + str(variant.split("/")[-1].replace("-", "_")))

forge/test/models/pytorch/multimodal/clip/test_clip.py

@@ -44,4 +44,6 @@ def test_clip_pytorch(test_device):
     text_model = CLIPTextWrapper(model)
     inputs = [inputs[0], inputs[2]]
 
-    compiled_model = forge.compile(text_model, sample_inputs=inputs, module_name="pt_clip_text_model")
+    compiled_model = forge.compile(
+        text_model, sample_inputs=inputs, module_name="pt_" + str(model_ckpt.split("/")[-1].replace("-", "_")) + "_text"
+    )

forge/test/models/pytorch/text/mistral/test_mistral.py

@@ -68,7 +68,11 @@ def test_mistral(variant, test_device):
     sample_inputs = tokenizer(prompt, return_tensors="pt")["input_ids"]
     inputs = [sample_inputs]
 
-    compiled_model = forge.compile(module, sample_inputs=inputs, module_name="pt_mistral")
+    compiled_model = forge.compile(
+        module,
+        sample_inputs=inputs,
+        module_name="pt_" + str(variant.split("/")[-1].replace("-", "_").replace(".", "_")),
+    )
 
 
 variants = ["mistralai/Mistral-7B-v0.1"]

scripts/model_analysis.py

@@ -125,17 +125,10 @@ def match_rule(self, exception: str):
             MatchingExceptionRule(
                 "Convert tt-forge attribute to an MLIR attribute", ["RuntimeError", "Unhandled attribute type"]
             ),
-            MatchingExceptionRule(
-                "ttmetal vs Forge Output Data mismatch",
-                ["AssertionError", "assert False", "where False = all([False])"],
-            ),
-            MatchingExceptionRule(
-                "Forge Verification Data mismatch",
-                ["ValueError", "forge/forge/verify/verify.py", "Data mismatch (compare_with_golden)"],
-            ),
+            MatchingExceptionRule("Runtime Datatype Unsupported", ["RuntimeError", "Unhandled dtype Bool"]),
             # Compiled model Runtime
             MatchingExceptionRule(
-                "Runtime Data mismatch", ["RuntimeError", "Tensor", "data type mismatch: expected", "got"]
+                "Runtime Datatype mismatch", ["RuntimeError", "Tensor", "data type mismatch: expected", "got"]
             ),
             MatchingExceptionRule(
                 "Runtime Shape mismatch", ["RuntimeError", "Tensor", "shape mismatch: expected", "got"]
@@ -190,6 +183,14 @@ def match_rule(self, exception: str):
     MatchingCompilerComponentException(
         CompilerComponent.TT_METAL,
         [
+            MatchingExceptionRule(
+                "TT-Metal vs Forge Output Data mismatch",
+                [
+                    "ValueError",
+                    "Data mismatch -> AutomaticValueChecker (compare_with_golden): framework_model",
+                    ", compiled_model",
+                ],
+            ),
             MatchingExceptionRule(
                 "ttnn.tilize validation",
                 [
@@ -334,6 +335,41 @@ def match_rule(self, exception: str):
                     "Only 2D, 3D, and 4D tensors are supported",
                 ],
             ),
+            MatchingExceptionRule(
+                "ttnn softmax",
+                [
+                    "RuntimeError",
+                    "tt-metal/ttnn/cpp/ttnn/operations/moreh/moreh_softmax/device/moreh_softmax_device_operation.cpp",
+                    "input.get_dtype() == DataType::BFLOAT16 || input.get_dtype() == DataType::BFLOAT8_B",
+                    "Inputs must be of bfloat16 or bfloat8_b type",
+                ],
+            ),
+            MatchingExceptionRule(
+                "ttnn unsqueeze_to_4D",
+                [
+                    "RuntimeError",
+                    "tt-metal/ttnn/cpp/ttnn/operations/core/core.cpp",
+                    "Tensor rank is greater than 4",
+                ],
+            ),
+            MatchingExceptionRule(
+                "ttnn matmul",
+                [
+                    "RuntimeError",
+                    "tt-metal/ttnn/cpp/ttnn/operations/matmul/device/matmul_op.cpp",
+                    "(input_tensor_a.get_legacy_shape()[-1] / in0_tile_shape[1]) % program_config.in0_block_w == 0",
+                    "Kt must be divisible by in0_block_w",
+                ],
+            ),
+            MatchingExceptionRule(
+                "tt-metal ncrisc build",
+                [
+                    "RuntimeError",
+                    "tt-metal/tt_metal/impl/program/program.cpp",
+                    "Failed to generate binaries for reader_conv_activations_padded_with_halo_3x3_weights_v2",
+                    "ncrisc build failed",
+                ],
+            ),
         ],
     ),
 ]