diff --git a/.github/workflows/model-analysis-weekly.yml b/.github/workflows/model-analysis-weekly.yml index ed9aabfec..cd520f1a1 100644 --- a/.github/workflows/model-analysis-weekly.yml +++ b/.github/workflows/model-analysis-weekly.yml @@ -74,6 +74,9 @@ jobs: cmake --build ${{ steps.strings.outputs.build-output-dir }} - name: Run Model Analysis Script + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_HUB_DISABLE_PROGRESS_BARS: 1 shell: bash run: | source env/activate diff --git a/forge/test/models/pytorch/audio/stereo/test_stereo.py b/forge/test/models/pytorch/audio/stereo/test_stereo.py index 340e4d070..b7b5fa834 100644 --- a/forge/test/models/pytorch/audio/stereo/test_stereo.py +++ b/forge/test/models/pytorch/audio/stereo/test_stereo.py @@ -32,4 +32,4 @@ def test_stereo(variant): compiled_model = forge.compile(framework_model, sample_inputs=inputs) - verify(inputs, framework_model, compiled_model) + verify(inputs, framework_model, compiled_model, module_name="pt_" + str(variant.split("/")[-1].replace("-", "_"))) diff --git a/forge/test/models/pytorch/multimodal/clip/test_clip.py b/forge/test/models/pytorch/multimodal/clip/test_clip.py index a19a8f690..16e52c65f 100644 --- a/forge/test/models/pytorch/multimodal/clip/test_clip.py +++ b/forge/test/models/pytorch/multimodal/clip/test_clip.py @@ -44,4 +44,6 @@ def test_clip_pytorch(test_device): text_model = CLIPTextWrapper(model) inputs = [inputs[0], inputs[2]] - compiled_model = forge.compile(text_model, sample_inputs=inputs, module_name="pt_clip_text_model") + compiled_model = forge.compile( + text_model, sample_inputs=inputs, module_name="pt_" + str(model_ckpt.split("/")[-1].replace("-", "_")) + "_text" + ) diff --git a/forge/test/models/pytorch/text/mistral/test_mistral.py b/forge/test/models/pytorch/text/mistral/test_mistral.py index a867c00e5..29e7ce4bd 100644 --- a/forge/test/models/pytorch/text/mistral/test_mistral.py +++ b/forge/test/models/pytorch/text/mistral/test_mistral.py @@ -68,7 +68,11 @@ def test_mistral(variant, test_device): sample_inputs = tokenizer(prompt, return_tensors="pt")["input_ids"] inputs = [sample_inputs] - compiled_model = forge.compile(module, sample_inputs=inputs, module_name="pt_mistral") + compiled_model = forge.compile( + module, + sample_inputs=inputs, + module_name="pt_" + str(variant.split("/")[-1].replace("-", "_").replace(".", "_")), + ) variants = ["mistralai/Mistral-7B-v0.1"] diff --git a/scripts/model_analysis.py b/scripts/model_analysis.py index 7acc71021..466129a65 100644 --- a/scripts/model_analysis.py +++ b/scripts/model_analysis.py @@ -125,14 +125,15 @@ def match_rule(self, exception: str): MatchingExceptionRule( "Convert tt-forge attribute to an MLIR attribute", ["RuntimeError", "Unhandled attribute type"] ), - MatchingExceptionRule( - "ttmetal vs Forge Output Data mismatch", - ["AssertionError", "assert False", "where False = all([False])"], - ), MatchingExceptionRule( "Forge Verification Data mismatch", - ["ValueError", "forge/forge/verify/verify.py", "Data mismatch (compare_with_golden)"], + [ + "ValueError", + "Data mismatch -> AutomaticValueChecker (compare_with_golden): framework_model", + ", compiled_model", + ], ), + MatchingExceptionRule("Runtime Datatype Unsupported", ["RuntimeError", "Unhandled dtype Bool"]), # Compiled model Runtime MatchingExceptionRule( "Runtime Data mismatch", ["RuntimeError", "Tensor", "data type mismatch: expected", "got"] @@ -334,6 +335,41 @@ def match_rule(self, exception: str): "Only 2D, 3D, and 4D tensors are supported", ], ), + MatchingExceptionRule( + "ttnn softmax", + [ + "RuntimeError", + "tt-metal/ttnn/cpp/ttnn/operations/moreh/moreh_softmax/device/moreh_softmax_device_operation.cpp", + "input.get_dtype() == DataType::BFLOAT16 || input.get_dtype() == DataType::BFLOAT8_B", + "Inputs must be of bfloat16 or bfloat8_b type", + ], + ), + MatchingExceptionRule( + "ttnn unsqueeze_to_4D", + [ + "RuntimeError", + "tt-metal/ttnn/cpp/ttnn/operations/core/core.cpp", + "Tensor rank is greater than 4", + ], + ), + MatchingExceptionRule( + "ttnn matmul", + [ + "RuntimeError", + "tt-metal/ttnn/cpp/ttnn/operations/matmul/device/matmul_op.cpp", + "(input_tensor_a.get_legacy_shape()[-1] / in0_tile_shape[1]) % program_config.in0_block_w == 0", + "Kt must be divisible by in0_block_w", + ], + ), + MatchingExceptionRule( + "tt-metal ncrisc build", + [ + "RuntimeError", + "tt-metal/tt_metal/impl/program/program.cpp", + "Failed to generate binaries for reader_conv_activations_padded_with_halo_3x3_weights_v2", + "ncrisc build failed", + ], + ), ], ), ]