From febad94618fbfbb9b355eafcfe59afd7db154519 Mon Sep 17 00:00:00 2001 From: pchandrasekaran Date: Mon, 2 Dec 2024 09:25:16 +0000 Subject: [PATCH] Add Weekly job for model analysis --- .github/workflows/build-image.yml | 51 ++++++++ .github/workflows/model-analysis-weekly.yml | 114 ++++++++++++++++++ .github/workflows/on-pr.yml | 10 +- forge/test/conftest.py | 21 ++++ .../pytorch/audio/whisper/test_whisper_0.py | 1 + .../pytorch/multimodal/clip/test_clip.py | 1 + .../pytorch/multimodal/vilt/test_vilt.py | 6 +- .../models/pytorch/text/albert/test_albert.py | 6 +- .../models/pytorch/text/bart/test_bart.py | 1 + .../models/pytorch/text/bert/test_bert.py | 4 + .../pytorch/text/codegen/test_codegen.py | 1 + .../text/distilbert/test_distilbert.py | 4 + .../test/models/pytorch/text/dpr/test_dpr.py | 3 + .../models/pytorch/text/falcon/test_falcon.py | 1 + .../models/pytorch/text/fuyu/test_fuyu_8b.py | 1 + .../pytorch/text/gemma/test_gemma_2b.py | 1 + .../models/pytorch/text/gpt2/test_gpt2.py | 1 + .../models/pytorch/text/gptneo/test_gptneo.py | 1 + .../models/pytorch/text/llama/test_llama3.py | 6 +- .../pytorch/text/mistral/test_mistral.py | 1 + .../test/models/pytorch/text/opt/test_opt.py | 3 + .../models/pytorch/text/phi2/test_phi2.py | 3 + .../models/pytorch/text/phi3/test_phi3.py | 3 + .../models/pytorch/text/qwen/test_qwen.py | 2 + .../pytorch/text/roberta/test_roberta.py | 2 + .../text/squeezebert/test_squeezebert.py | 1 + forge/test/models/pytorch/text/t5/test_t5.py | 3 +- .../models/pytorch/text/xglm/test_xglm.py | 1 + .../pytorch/timeseries/nbeats/test_nbeats.py | 9 +- .../pytorch/vision/alexnet/test_alexnet.py | 2 + .../vision/autoencoder/test_autoencoder.py | 2 + .../models/pytorch/vision/deit/test_deit.py | 3 +- .../pytorch/vision/densenet/test_densenet.py | 6 +- .../models/pytorch/vision/dla/test_dla.py | 3 +- .../vision/efficientnet/test_efficientnet.py | 4 +- .../models/pytorch/vision/fpn/test_fpn.py | 1 + .../pytorch/vision/ghostnet/test_ghostnet.py | 3 +- .../vision/googlenet/test_googlenet.py | 1 + .../models/pytorch/vision/hrnet/test_hrnet.py | 6 +- .../vision/inception/test_inception_v4.py | 2 + .../vision/mlp_mixer/test_mlp_mixer.py | 3 +- .../vision/mobilenet/test_mobilenet_v1.py | 3 + .../vision/mobilenet/test_mobilenet_v2.py | 8 +- .../vision/mobilenet/test_mobilenet_v3.py | 6 +- .../pytorch/vision/monodle/test_monodle.py | 1 + .../vision/perceiverio/test_perceiverio.py | 3 +- .../models/pytorch/vision/rcnn/test_rcnn.py | 1 + .../pytorch/vision/resnet/test_resnet.py | 2 + .../pytorch/vision/resnext/test_resnext.py | 7 ++ .../vision/retinanet/test_retinanet.py | 3 +- .../vision/segformer/test_segformer.py | 6 +- .../ssd300_resnet50/test_ssd300_resnet50.py | 1 + .../models/pytorch/vision/swin/test_swin.py | 1 + .../models/pytorch/vision/unet/test_unet.py | 3 + .../models/pytorch/vision/vgg/test_vgg.py | 6 +- .../models/pytorch/vision/vit/test_vit.py | 3 +- .../pytorch/vision/vovnet/test_vovnet.py | 10 +- .../vision/wideresnet/test_wideresnet.py | 6 +- .../pytorch/vision/xception/test_xception.py | 3 +- .../pytorch/vision/yolo/test_yolo_v5.py | 10 +- .../pytorch/vision/yolo/test_yolo_v6.py | 3 +- .../models/pytorch/vision/yolo/test_yolox.py | 3 +- scripts/model_analysis.py | 103 ++++++++-------- 63 files changed, 393 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/build-image.yml create mode 100644 .github/workflows/model-analysis-weekly.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 000000000..2613b33aa --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,51 @@ +name: Build Docker Image + +on: + workflow_dispatch: + workflow_call: + outputs: + docker-image: + description: "Built docker image name" + value: ${{ jobs.build-image.outputs.docker-image }} + + +jobs: + + build-image: + runs-on: builder + outputs: + docker-image: ${{ steps.build.outputs.docker-image }} + steps: + - name: Fix permissions + shell: bash + run: sudo chown ubuntu:ubuntu -R $(pwd) + + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 # Fetch all history and tags + + # Clean everything from submodules (needed to avoid issues + # with cmake generated files leftover from previous builds) + - name: Cleanup submodules + run: | + git submodule foreach --recursive git clean -ffdx + git submodule foreach --recursive git reset --hard + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Docker images and output the image name + id: build + shell: bash + run: | + # Output the image name + set pipefail + .github/build-docker-images.sh | tee docker.log + DOCKER_CI_IMAGE=$(tail -n 1 docker.log) + echo "DOCKER_CI_IMAGE $DOCKER_CI_IMAGE" + echo "docker-image=$DOCKER_CI_IMAGE" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/model-analysis-weekly.yml b/.github/workflows/model-analysis-weekly.yml new file mode 100644 index 000000000..fca1393ac --- /dev/null +++ b/.github/workflows/model-analysis-weekly.yml @@ -0,0 +1,114 @@ +name: Model Analysis Weekly + +on: + workflow_dispatch: + # schedule: + # - cron: '0 23 * * 5' # 11:00 PM UTC Friday (12:00 AM Saturday Serbia) + push: + branches: ["pchandrasekaran/model_analysis_weekly_job"] + +jobs: + + docker-build: + uses: ./.github/workflows/build-image.yml + secrets: inherit + + model-analysis: + needs: docker-build + runs-on: runner + + container: + image: ${{ needs.docker-build.outputs.docker-image }} + options: --device /dev/tenstorrent/0 + volumes: + - /dev/hugepages:/dev/hugepages + - /dev/hugepages-1G:/dev/hugepages-1G + - /etc/udev/rules.d:/etc/udev/rules.d + - /lib/modules:/lib/modules + - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env + + steps: + + - name: Set reusable strings + id: strings + shell: bash + run: | + echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT" + echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT" + + - name: Git safe dir + run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }} + + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 # Fetch all history and tags + + # Clean everything from submodules (needed to avoid issues + # with cmake generated files leftover from previous builds) + - name: Cleanup submodules + run: | + git submodule foreach --recursive git clean -ffdx + git submodule foreach --recursive git reset --hard + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + create-symlink: true + key: model-analysis-${{ runner.os }} + + - name: Build + shell: bash + run: | + source env/activate + cmake -G Ninja \ + -B ${{ steps.strings.outputs.build-output-dir }} \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + cmake --build ${{ steps.strings.outputs.build-output-dir }} + + - name: Run Model Analysis Script + shell: bash + run: | + source env/activate + apt install -y libgl1-mesa-glx + python scripts/model_analysis.py \ + --test_directory_or_file_path forge/test/models/pytorch \ + --dump_failure_logs \ + --markdown_directory_path ./model_analysis_docs \ + --unique_ops_output_directory_path ./models_unique_ops_output \ + 2>&1 | tee model_analysis.log + + - name: Upload Model Analysis Script Logs + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: model-analysis-outputs + path: model_analysis.log + + - name: Upload Models Unique Ops test Failure Logs + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: unique-ops-logs + path: ./models_unique_ops_output + + # - name: Create Pull Request + # uses: peter-evans/create-pull-request@v7 + # with: + # branch: model_analysis + # committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + # author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + # base: main + # commit-message: "Update model analysis docs" + # title: "Update model analysis docs" + # body: "This PR will update model analysis docs" + # labels: model_analysis + # delete-branch: true + # token: ${{ secrets.GH_TOKEN }} + # add-paths: | + # model_analysis_docs/ + # draft: true # Need to remove diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 34a567df1..f01a6e2a0 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -12,8 +12,8 @@ jobs: pre-commit: uses: ./.github/workflows/pre-commit.yml secrets: inherit - docker-build: - uses: ./.github/workflows/build-and-test.yml - secrets: inherit - with: - test_mark: 'push' + # docker-build: + # uses: ./.github/workflows/build-and-test.yml + # secrets: inherit + # with: + # test_mark: 'push' diff --git a/forge/test/conftest.py b/forge/test/conftest.py index 509858d55..f7e4d6f52 100644 --- a/forge/test/conftest.py +++ b/forge/test/conftest.py @@ -448,3 +448,24 @@ def pytest_runtest_logreport(report): for key, default_value in environ_before_test.items(): if os.environ.get(key, "") != default_value: os.environ[key] = default_value + + +def pytest_collection_modifyitems(config, items): + + marker = config.getoption("-m") # Get the marker from the -m option + + if marker and marker == "model_analysis_new": # If a marker is specified + filtered_items = [item for item in items if marker in item.keywords] + print("Automatic Model Analysis Collected tests: ") + test_count = 0 + for item in items: + if marker in item.keywords: + test_file_path = item.location[0] + test_name = item.location[2] + print(f"{test_file_path}::{test_name}") + test_count += 1 + print(f"Automatic Model Analysis Collected test count: {test_count}") + if not filtered_items: # Warn if no tests match the marker + print(f"Warning: No tests found with marker '{marker}'.") + else: + print(items) diff --git a/forge/test/models/pytorch/audio/whisper/test_whisper_0.py b/forge/test/models/pytorch/audio/whisper/test_whisper_0.py index 89b84a705..e9c50b8d1 100644 --- a/forge/test/models/pytorch/audio/whisper/test_whisper_0.py +++ b/forge/test/models/pytorch/audio/whisper/test_whisper_0.py @@ -99,6 +99,7 @@ def forward(self, decoder_input_ids, encoder_hidden_states): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_whisper(test_device, variant): diff --git a/forge/test/models/pytorch/multimodal/clip/test_clip.py b/forge/test/models/pytorch/multimodal/clip/test_clip.py index 810759ee6..a19a8f690 100644 --- a/forge/test/models/pytorch/multimodal/clip/test_clip.py +++ b/forge/test/models/pytorch/multimodal/clip/test_clip.py @@ -17,6 +17,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_clip_pytorch(test_device): # Set Forge configuration parameters diff --git a/forge/test/models/pytorch/multimodal/vilt/test_vilt.py b/forge/test/models/pytorch/multimodal/vilt/test_vilt.py index 313327332..96aeedbaa 100644 --- a/forge/test/models/pytorch/multimodal/vilt/test_vilt.py +++ b/forge/test/models/pytorch/multimodal/vilt/test_vilt.py @@ -50,8 +50,9 @@ def generate_model_vilt_question_answering_hf_pytorch(test_device, variant): variants = ["dandelin/vilt-b32-finetuned-vqa"] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_vilt_question_answering_hf_pytorch(variant, test_device): model, inputs, _ = generate_model_vilt_question_answering_hf_pytorch( test_device, @@ -93,9 +94,10 @@ def generate_model_vilt_maskedlm_hf_pytorch(test_device, variant): variants = ["dandelin/vilt-b32-mlm"] +@pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="pcc=0.9498278562793674") @pytest.mark.parametrize("variant", variants, ids=variants) -@pytest.mark.nightly def test_vilt_maskedlm_hf_pytorch(variant, test_device): model, inputs, _ = generate_model_vilt_maskedlm_hf_pytorch( test_device, diff --git a/forge/test/models/pytorch/text/albert/test_albert.py b/forge/test/models/pytorch/text/albert/test_albert.py index 13af28d5e..d9d27ef3c 100644 --- a/forge/test/models/pytorch/text/albert/test_albert.py +++ b/forge/test/models/pytorch/text/albert/test_albert.py @@ -12,10 +12,11 @@ variants = ["v1", "v2"] +@pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.parametrize("size", sizes, ids=sizes) -@pytest.mark.nightly def test_albert_masked_lm_pytorch(size, variant, test_device): model_ckpt = f"albert-{size}-{variant}" @@ -55,10 +56,11 @@ def test_albert_masked_lm_pytorch(size, variant, test_device): variants = ["v1", "v2"] +@pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.parametrize("size", sizes, ids=sizes) -@pytest.mark.nightly def test_albert_token_classification_pytorch(size, variant, test_device): compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/text/bart/test_bart.py b/forge/test/models/pytorch/text/bart/test_bart.py index 301a20374..02ecad419 100644 --- a/forge/test/models/pytorch/text/bart/test_bart.py +++ b/forge/test/models/pytorch/text/bart/test_bart.py @@ -23,6 +23,7 @@ def forward(self, input_ids, attention_mask, decoder_input_ids): @pytest.mark.nightly +@pytest.mark.model_analysis def test_pt_bart_classifier(test_device): compiler_cfg = _get_global_compiler_config() compiler_cfg.compile_depth = CompileDepth.SPLIT_GRAPH diff --git a/forge/test/models/pytorch/text/bert/test_bert.py b/forge/test/models/pytorch/text/bert/test_bert.py index a7f8b0109..1877e0a6a 100644 --- a/forge/test/models/pytorch/text/bert/test_bert.py +++ b/forge/test/models/pytorch/text/bert/test_bert.py @@ -40,6 +40,7 @@ def generate_model_bert_maskedlm_hf_pytorch(variant): @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.nightly +@pytest.mark.model_analysis def test_bert_masked_lm_pytorch(test_device): model, inputs, _ = generate_model_bert_maskedlm_hf_pytorch("bert-base-uncased") @@ -89,6 +90,7 @@ def generate_model_bert_qa_hf_pytorch(variant): @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.nightly +@pytest.mark.model_analysis def test_bert_question_answering_pytorch(test_device): model, inputs, _ = generate_model_bert_qa_hf_pytorch("bert-large-cased-whole-word-masking-finetuned-squad") @@ -128,6 +130,7 @@ def generate_model_bert_seqcls_hf_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_bert_sequence_classification_pytorch(test_device): model, inputs, _ = generate_model_bert_seqcls_hf_pytorch( "textattack/bert-base-uncased-SST-2", @@ -169,6 +172,7 @@ def generate_model_bert_tkcls_hf_pytorch(variant): @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.nightly +@pytest.mark.model_analysis def test_bert_token_classification_pytorch(test_device): model, inputs, _ = generate_model_bert_tkcls_hf_pytorch("dbmdz/bert-large-cased-finetuned-conll03-english") diff --git a/forge/test/models/pytorch/text/codegen/test_codegen.py b/forge/test/models/pytorch/text/codegen/test_codegen.py index 82c3115da..15fc3acf4 100644 --- a/forge/test/models/pytorch/text/codegen/test_codegen.py +++ b/forge/test/models/pytorch/text/codegen/test_codegen.py @@ -21,6 +21,7 @@ @pytest.mark.xfail(reason="RuntimeError: Tensor 41 - data type mismatch: expected Float32, got BFloat16") @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_codegen(test_device, variant): # Configurations diff --git a/forge/test/models/pytorch/text/distilbert/test_distilbert.py b/forge/test/models/pytorch/text/distilbert/test_distilbert.py index 80362774d..299ad2be2 100644 --- a/forge/test/models/pytorch/text/distilbert/test_distilbert.py +++ b/forge/test/models/pytorch/text/distilbert/test_distilbert.py @@ -16,6 +16,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_distilbert_masked_lm_pytorch(variant, test_device): # Load DistilBert tokenizer and model from HuggingFace @@ -46,6 +47,7 @@ def test_distilbert_masked_lm_pytorch(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_distilbert_question_answering_pytorch(test_device): # Load Bert tokenizer and model from HuggingFace model_ckpt = "distilbert-base-cased-distilled-squad" @@ -82,6 +84,7 @@ def test_distilbert_question_answering_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_distilbert_sequence_classification_pytorch(test_device): # Load DistilBert tokenizer and model from HuggingFace @@ -109,6 +112,7 @@ def test_distilbert_sequence_classification_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_distilbert_token_classification_pytorch(test_device): # Load DistilBERT tokenizer and model from HuggingFace model_ckpt = "Davlan/distilbert-base-multilingual-cased-ner-hrl" diff --git a/forge/test/models/pytorch/text/dpr/test_dpr.py b/forge/test/models/pytorch/text/dpr/test_dpr.py index cd3fcce1f..881f1de53 100644 --- a/forge/test/models/pytorch/text/dpr/test_dpr.py +++ b/forge/test/models/pytorch/text/dpr/test_dpr.py @@ -20,6 +20,7 @@ @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_dpr_context_encoder_pytorch(variant, test_device): @@ -61,6 +62,7 @@ def test_dpr_context_encoder_pytorch(variant, test_device): @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_dpr_question_encoder_pytorch(variant, test_device): # Load Bert tokenizer and model from HuggingFace @@ -101,6 +103,7 @@ def test_dpr_question_encoder_pytorch(variant, test_device): @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_dpr_reader_pytorch(variant, test_device): # Load Bert tokenizer and model from HuggingFace diff --git a/forge/test/models/pytorch/text/falcon/test_falcon.py b/forge/test/models/pytorch/text/falcon/test_falcon.py index 9493740fd..9a5703064 100644 --- a/forge/test/models/pytorch/text/falcon/test_falcon.py +++ b/forge/test/models/pytorch/text/falcon/test_falcon.py @@ -8,6 +8,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_falcon(test_device): compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/text/fuyu/test_fuyu_8b.py b/forge/test/models/pytorch/text/fuyu/test_fuyu_8b.py index dc14e4dde..356bafe93 100644 --- a/forge/test/models/pytorch/text/fuyu/test_fuyu_8b.py +++ b/forge/test/models/pytorch/text/fuyu/test_fuyu_8b.py @@ -30,6 +30,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_fuyu8b(test_device): # Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/text/gemma/test_gemma_2b.py b/forge/test/models/pytorch/text/gemma/test_gemma_2b.py index 58dd095bd..3b40abe6f 100644 --- a/forge/test/models/pytorch/text/gemma/test_gemma_2b.py +++ b/forge/test/models/pytorch/text/gemma/test_gemma_2b.py @@ -254,6 +254,7 @@ def forward(self, hidden_states, attn_mask, pos_ids): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_gemma_2b(test_device, variant): # Random see for reproducibility diff --git a/forge/test/models/pytorch/text/gpt2/test_gpt2.py b/forge/test/models/pytorch/text/gpt2/test_gpt2.py index e5c976464..7934d2913 100644 --- a/forge/test/models/pytorch/text/gpt2/test_gpt2.py +++ b/forge/test/models/pytorch/text/gpt2/test_gpt2.py @@ -12,6 +12,7 @@ @pytest.mark.xfail(reason="RuntimeError: Tensor 6 - data type mismatch: expected Float32, got BFloat16") @pytest.mark.nightly +@pytest.mark.model_analysis def test_gpt2_text_gen(test_device): # Load tokenizer and model from HuggingFace config = GPT2Config.from_pretrained("gpt2") diff --git a/forge/test/models/pytorch/text/gptneo/test_gptneo.py b/forge/test/models/pytorch/text/gptneo/test_gptneo.py index e47fbb2f1..b6328f072 100644 --- a/forge/test/models/pytorch/text/gptneo/test_gptneo.py +++ b/forge/test/models/pytorch/text/gptneo/test_gptneo.py @@ -23,6 +23,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_gptneo_causal_lm(variant, test_device): # Set random seed for repeatability diff --git a/forge/test/models/pytorch/text/llama/test_llama3.py b/forge/test/models/pytorch/text/llama/test_llama3.py index 3f9916e16..7bd8fecfe 100644 --- a/forge/test/models/pytorch/text/llama/test_llama3.py +++ b/forge/test/models/pytorch/text/llama/test_llama3.py @@ -109,8 +109,9 @@ def _update_causal_mask( LlamaModel._update_causal_mask = _update_causal_mask -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_llama3_causal_lm(variant, test_device): # Configurations compiler_cfg = forge.config._get_global_compiler_config() @@ -151,8 +152,9 @@ def test_llama3_causal_lm(variant, test_device): ) -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_llama3_sequence_classification(variant, test_device): # Configurations diff --git a/forge/test/models/pytorch/text/mistral/test_mistral.py b/forge/test/models/pytorch/text/mistral/test_mistral.py index 40b56dbc7..a867c00e5 100644 --- a/forge/test/models/pytorch/text/mistral/test_mistral.py +++ b/forge/test/models/pytorch/text/mistral/test_mistral.py @@ -43,6 +43,7 @@ def test_mistral_decoder_layer(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_mistral(variant, test_device): diff --git a/forge/test/models/pytorch/text/opt/test_opt.py b/forge/test/models/pytorch/text/opt/test_opt.py index a0bcbbad4..69a286d11 100644 --- a/forge/test/models/pytorch/text/opt/test_opt.py +++ b/forge/test/models/pytorch/text/opt/test_opt.py @@ -10,6 +10,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_opt_causal_lm(variant, test_device): # Load tokenizer and model from HuggingFace @@ -46,6 +47,7 @@ def test_opt_causal_lm(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_opt_qa(variant, test_device): # Load tokenizer and model from HuggingFace @@ -81,6 +83,7 @@ def test_opt_qa(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_opt_sequence_classification(variant, test_device): # Set Forge configuration parameters diff --git a/forge/test/models/pytorch/text/phi2/test_phi2.py b/forge/test/models/pytorch/text/phi2/test_phi2.py index c4c51c217..f4e657a0a 100644 --- a/forge/test/models/pytorch/text/phi2/test_phi2.py +++ b/forge/test/models/pytorch/text/phi2/test_phi2.py @@ -17,6 +17,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.xfail(reason="weights.get_dtype() == DataType::BFLOAT16") def test_phi2_clm(variant, test_device): @@ -66,6 +67,7 @@ def test_phi2_clm(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants) @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") def test_phi2_token_classification(variant, test_device): @@ -106,6 +108,7 @@ def test_phi2_token_classification(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants) @pytest.mark.xfail(reason="TT_FATAL(weights.get_dtype() == DataType::BFLOAT16) in embedding op") def test_phi2_sequence_classification(variant, test_device): diff --git a/forge/test/models/pytorch/text/phi3/test_phi3.py b/forge/test/models/pytorch/text/phi3/test_phi3.py index 2a82fa6e6..bcc32df1f 100644 --- a/forge/test/models/pytorch/text/phi3/test_phi3.py +++ b/forge/test/models/pytorch/text/phi3/test_phi3.py @@ -17,6 +17,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants) def test_phi3_causal_lm(variant, test_device): @@ -61,6 +62,7 @@ def test_phi3_causal_lm(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="RuntimeError: Indices tensor must be in row major layout.") @pytest.mark.parametrize("variant", variants) def test_phi3_token_classification(variant, test_device): @@ -101,6 +103,7 @@ def test_phi3_token_classification(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="RuntimeError: Embedding Device Operation Layout Mismatch - Expected ROW_MAJOR") @pytest.mark.parametrize("variant", variants) def test_phi3_sequence_classification(variant, test_device): diff --git a/forge/test/models/pytorch/text/qwen/test_qwen.py b/forge/test/models/pytorch/text/qwen/test_qwen.py index 1b1311066..06584eeb2 100644 --- a/forge/test/models/pytorch/text/qwen/test_qwen.py +++ b/forge/test/models/pytorch/text/qwen/test_qwen.py @@ -9,6 +9,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_qwen1_5_causal_lm(test_device): # Set PyBuda configurations @@ -58,6 +59,7 @@ def parse_chat_completion(text: str): @pytest.mark.nightly +@pytest.mark.model_analysis def test_qwen1_5_chat(test_device): # Set PyBuda configurations diff --git a/forge/test/models/pytorch/text/roberta/test_roberta.py b/forge/test/models/pytorch/text/roberta/test_roberta.py index c156d8d3b..7c78f1af4 100644 --- a/forge/test/models/pytorch/text/roberta/test_roberta.py +++ b/forge/test/models/pytorch/text/roberta/test_roberta.py @@ -9,6 +9,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_roberta_masked_lm(test_device): # Load Albert tokenizer and model from HuggingFace tokenizer = download_model(AutoTokenizer.from_pretrained, "xlm-roberta-base") @@ -34,6 +35,7 @@ def test_roberta_masked_lm(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_roberta_sentiment_pytorch(test_device): # Load Bart tokenizer and model from HuggingFace tokenizer = download_model(AutoTokenizer.from_pretrained, "cardiffnlp/twitter-roberta-base-sentiment") diff --git a/forge/test/models/pytorch/text/squeezebert/test_squeezebert.py b/forge/test/models/pytorch/text/squeezebert/test_squeezebert.py index ed32a331e..1016f0c1a 100644 --- a/forge/test/models/pytorch/text/squeezebert/test_squeezebert.py +++ b/forge/test/models/pytorch/text/squeezebert/test_squeezebert.py @@ -8,6 +8,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_squeezebert_sequence_classification_pytorch(test_device): # Load Bart tokenizer and model from HuggingFace tokenizer = download_model(AutoTokenizer.from_pretrained, "squeezebert/squeezebert-mnli") diff --git a/forge/test/models/pytorch/text/t5/test_t5.py b/forge/test/models/pytorch/text/t5/test_t5.py index dc5513705..d0faa8172 100644 --- a/forge/test/models/pytorch/text/t5/test_t5.py +++ b/forge/test/models/pytorch/text/t5/test_t5.py @@ -95,8 +95,9 @@ def forward(self, decoder_input_ids, encoder_outputs): ] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_t5_generation(variant, test_device): compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/text/xglm/test_xglm.py b/forge/test/models/pytorch/text/xglm/test_xglm.py index 823029496..6c5410f0a 100644 --- a/forge/test/models/pytorch/text/xglm/test_xglm.py +++ b/forge/test/models/pytorch/text/xglm/test_xglm.py @@ -11,6 +11,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_xglm_causal_lm(variant, test_device): # Set Forge configuration parameters diff --git a/forge/test/models/pytorch/timeseries/nbeats/test_nbeats.py b/forge/test/models/pytorch/timeseries/nbeats/test_nbeats.py index 60a7b18fb..ccddc13b2 100644 --- a/forge/test/models/pytorch/timeseries/nbeats/test_nbeats.py +++ b/forge/test/models/pytorch/timeseries/nbeats/test_nbeats.py @@ -17,7 +17,8 @@ @pytest.mark.nightly -# @pytest.mark.xfail(reason="Failing with pcc=0.82") +@pytest.mark.model_analysis +@pytest.mark.xfail(reason="Failing with pcc=0.82") def test_nbeats_with_seasonality_basis(test_device): compiler_cfg = forge.config._get_global_compiler_config() @@ -44,7 +45,8 @@ def test_nbeats_with_seasonality_basis(test_device): @pytest.mark.nightly -# @pytest.mark.xfail(reason="Failing with pcc=0.83") +@pytest.mark.model_analysis +@pytest.mark.xfail(reason="Failing with pcc=0.83") def test_nbeats_with_generic_basis(test_device): compiler_cfg = forge.config._get_global_compiler_config() @@ -65,7 +67,8 @@ def test_nbeats_with_generic_basis(test_device): @pytest.mark.nightly -# @pytest.mark.xfail(eason="Failing with pcc=0.83") +@pytest.mark.model_analysis +@pytest.mark.xfail(reason="Failing with pcc=0.83") def test_nbeats_with_trend_basis(test_device): compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/vision/alexnet/test_alexnet.py b/forge/test/models/pytorch/vision/alexnet/test_alexnet.py index 799db7c93..9034c0aee 100644 --- a/forge/test/models/pytorch/vision/alexnet/test_alexnet.py +++ b/forge/test/models/pytorch/vision/alexnet/test_alexnet.py @@ -14,6 +14,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_alexnet_torchhub(test_device): # Configurations compiler_cfg = forge.config._get_global_compiler_config() @@ -47,6 +48,7 @@ def test_alexnet_torchhub(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_alexnet_osmr(test_device): # Configurations compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py b/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py index d35dfc6c8..b980f0b46 100644 --- a/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py +++ b/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py @@ -13,6 +13,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_conv_ae_pytorch(test_device): # Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() @@ -40,6 +41,7 @@ def test_conv_ae_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis_new def test_linear_ae_pytorch(test_device): # Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/vision/deit/test_deit.py b/forge/test/models/pytorch/vision/deit/test_deit.py index ac94cb375..33c7f1cb4 100644 --- a/forge/test/models/pytorch/vision/deit/test_deit.py +++ b/forge/test/models/pytorch/vision/deit/test_deit.py @@ -41,8 +41,9 @@ def generate_model_deit_imgcls_hf_pytorch(variant): ] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_vit_base_classify_224_hf_pytorch(variant, test_device): model, inputs, _ = generate_model_deit_imgcls_hf_pytorch( variant, diff --git a/forge/test/models/pytorch/vision/densenet/test_densenet.py b/forge/test/models/pytorch/vision/densenet/test_densenet.py index baf624fad..5a5084c93 100644 --- a/forge/test/models/pytorch/vision/densenet/test_densenet.py +++ b/forge/test/models/pytorch/vision/densenet/test_densenet.py @@ -14,8 +14,9 @@ variants = ["densenet121", "densenet121_hf_xray"] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_densenet_121_pytorch(variant, test_device): # STEP 1: Set Forge configuration parameters @@ -42,6 +43,7 @@ def test_densenet_121_pytorch(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_densenet_161_pytorch(test_device): # STEP 1: Set Forge configuration parameters @@ -59,6 +61,7 @@ def test_densenet_161_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_densenet_169_pytorch(test_device): # STEP 1: Set Forge configuration parameters @@ -76,6 +79,7 @@ def test_densenet_169_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_densenet_201_pytorch(test_device): # STEP 1: Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/dla/test_dla.py b/forge/test/models/pytorch/vision/dla/test_dla.py index 48dbc3584..b0e791068 100644 --- a/forge/test/models/pytorch/vision/dla/test_dla.py +++ b/forge/test/models/pytorch/vision/dla/test_dla.py @@ -36,8 +36,9 @@ variants = list(variants_func.keys()) -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_dla_pytorch(variant, test_device): # Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/vision/efficientnet/test_efficientnet.py b/forge/test/models/pytorch/vision/efficientnet/test_efficientnet.py index 7d77da435..4380c804e 100644 --- a/forge/test/models/pytorch/vision/efficientnet/test_efficientnet.py +++ b/forge/test/models/pytorch/vision/efficientnet/test_efficientnet.py @@ -44,8 +44,9 @@ ] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_efficientnet_timm(variant, test_device): # Configuration @@ -102,6 +103,7 @@ def get_state_dict(self, *args, **kwargs): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.parametrize("variant", variants) @pytest.mark.xfail(reason="Runtime Error: Reshape Operation Fails Due to Mismatched Tensor Volume") def test_efficientnet_torchvision(variant, test_device): diff --git a/forge/test/models/pytorch/vision/fpn/test_fpn.py b/forge/test/models/pytorch/vision/fpn/test_fpn.py index 8571a3fe5..b32ffbd73 100644 --- a/forge/test/models/pytorch/vision/fpn/test_fpn.py +++ b/forge/test/models/pytorch/vision/fpn/test_fpn.py @@ -8,6 +8,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis_new def test_fpn_pytorch(test_device): compiler_cfg = forge.config._get_global_compiler_config() compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH diff --git a/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py b/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py index fea73d009..5bc6b45bb 100644 --- a/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py +++ b/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py @@ -19,9 +19,10 @@ variants = ["ghostnet_100"] +@pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="Runtime error : Invalid arguments to reshape") @pytest.mark.parametrize("variant", variants, ids=variants) -@pytest.mark.nightly def test_ghostnet_timm(variant, test_device): # STEP 1: Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/googlenet/test_googlenet.py b/forge/test/models/pytorch/vision/googlenet/test_googlenet.py index e9225d059..58a5408b7 100644 --- a/forge/test/models/pytorch/vision/googlenet/test_googlenet.py +++ b/forge/test/models/pytorch/vision/googlenet/test_googlenet.py @@ -12,6 +12,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_googlenet_pytorch(test_device): # Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object diff --git a/forge/test/models/pytorch/vision/hrnet/test_hrnet.py b/forge/test/models/pytorch/vision/hrnet/test_hrnet.py index 28bbabdf8..959c81da5 100644 --- a/forge/test/models/pytorch/vision/hrnet/test_hrnet.py +++ b/forge/test/models/pytorch/vision/hrnet/test_hrnet.py @@ -82,8 +82,9 @@ def generate_model_hrnet_imgcls_osmr_pytorch(variant): ] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_hrnet_osmr_pytorch(test_device, variant): model, inputs, _ = generate_model_hrnet_imgcls_osmr_pytorch( variant, @@ -147,8 +148,9 @@ def generate_model_hrnet_imgcls_timm_pytorch(variant): ] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_hrnet_timm_pytorch(test_device, variant): model, inputs, _ = generate_model_hrnet_imgcls_timm_pytorch( variant, diff --git a/forge/test/models/pytorch/vision/inception/test_inception_v4.py b/forge/test/models/pytorch/vision/inception/test_inception_v4.py index 9be3df367..69927e39a 100644 --- a/forge/test/models/pytorch/vision/inception/test_inception_v4.py +++ b/forge/test/models/pytorch/vision/inception/test_inception_v4.py @@ -32,6 +32,7 @@ def generate_model_inceptionV4_imgcls_osmr_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_inception_v4_osmr_pytorch(test_device): model, inputs = generate_model_inceptionV4_imgcls_osmr_pytorch("inceptionv4") compiled_model = forge.compile(model, sample_inputs=inputs, module_name="pt_osmr_inception_v4") @@ -48,6 +49,7 @@ def generate_model_inceptionV4_imgcls_timm_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_inception_v4_timm_pytorch(test_device): model, inputs = generate_model_inceptionV4_imgcls_timm_pytorch("inception_v4") diff --git a/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py b/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py index 83072bb98..b4f4d3c3c 100644 --- a/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py +++ b/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py @@ -30,8 +30,9 @@ ] -@pytest.mark.parametrize("variant", varaints, ids=varaints) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", varaints, ids=varaints) def test_mlp_mixer_timm_pytorch(variant, test_device): model = download_model(timm.create_model, variant, pretrained=True) diff --git a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py index 20bdd9d4d..b4d85a698 100644 --- a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py +++ b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py @@ -30,6 +30,7 @@ def generate_model_mobilenetV1_base_custom_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="RuntimeError: Invalid arguments to reshape") def test_mobilenetv1_basic(test_device): model, inputs, _ = generate_model_mobilenetV1_base_custom_pytorch( @@ -68,6 +69,7 @@ def generate_model_mobilenetv1_imgcls_hf_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_mobilenetv1_192(test_device): model, inputs, _ = generate_model_mobilenetv1_imgcls_hf_pytorch( test_device, @@ -96,6 +98,7 @@ def generate_model_mobilenetV1I224_imgcls_hf_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_mobilenetv1_224(test_device): model, inputs, _ = generate_model_mobilenetV1I224_imgcls_hf_pytorch( test_device, diff --git a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py index 46d01b01b..222e668ff 100644 --- a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py +++ b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py @@ -41,6 +41,7 @@ def generate_model_mobilenetV2_imgcls_torchhub_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_mobilenetv2_basic(test_device): model, inputs, _ = generate_model_mobilenetV2_imgcls_torchhub_pytorch( test_device, @@ -67,6 +68,7 @@ def generate_model_mobilenetV2I96_imgcls_hf_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_mobilenetv2_96(test_device): model, inputs, _ = generate_model_mobilenetV2I96_imgcls_hf_pytorch( test_device, @@ -93,6 +95,7 @@ def generate_model_mobilenetV2I160_imgcls_hf_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_mobilenetv2_160(test_device): model, inputs, _ = generate_model_mobilenetV2I160_imgcls_hf_pytorch( test_device, @@ -121,6 +124,7 @@ def generate_model_mobilenetV2I244_imgcls_hf_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_mobilenetv2_224(test_device): model, inputs, _ = generate_model_mobilenetV2I244_imgcls_hf_pytorch( test_device, @@ -158,6 +162,7 @@ def generate_model_mobilenetV2_imgcls_timm_pytorch(test_device, variant): # @pytest.mark.xfail(reason="Runtime error : Invalid arguments to reshape") @pytest.mark.nightly +@pytest.mark.model_analysis def test_mobilenetv2_timm(test_device): model, inputs, _ = generate_model_mobilenetV2_imgcls_timm_pytorch( test_device, @@ -209,8 +214,9 @@ def generate_model_mobilenetV2_semseg_hf_pytorch(test_device, variant): variants = ["google/deeplabv3_mobilenet_v2_1.0_513"] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_mobilenetv2_deeplabv3(variant, test_device): model, inputs, _ = generate_model_mobilenetV2_semseg_hf_pytorch( test_device, diff --git a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py index 339b6c534..6579f94f7 100644 --- a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py +++ b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py @@ -40,9 +40,10 @@ def generate_model_mobilenetV3_imgcls_torchhub_pytorch(test_device, variant): variants = ["mobilenet_v3_large", "mobilenet_v3_small"] +@pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="Runtime error : Invalid arguments to reshape") @pytest.mark.parametrize("variant", variants, ids=variants) -@pytest.mark.nightly def test_mobilenetv3_basic(variant, test_device): model, inputs, _ = generate_model_mobilenetV3_imgcls_torchhub_pytorch( test_device, @@ -93,9 +94,10 @@ def generate_model_mobilenetV3_imgcls_timm_pytorch(test_device, variant): variants = ["mobilenetv3_large_100", "mobilenetv3_small_100"] +@pytest.mark.nightly +@pytest.mark.model_analysis @pytest.mark.xfail(reason="Runtime error : Invalid arguments to reshape") @pytest.mark.parametrize("variant", variants, ids=variants) -@pytest.mark.nightly def test_mobilenetv3_timm(variant, test_device): model, inputs, _ = generate_model_mobilenetV3_imgcls_timm_pytorch( test_device, diff --git a/forge/test/models/pytorch/vision/monodle/test_monodle.py b/forge/test/models/pytorch/vision/monodle/test_monodle.py index 2eec7dbae..0de6b679e 100644 --- a/forge/test/models/pytorch/vision/monodle/test_monodle.py +++ b/forge/test/models/pytorch/vision/monodle/test_monodle.py @@ -11,6 +11,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_monodle_pytorch(test_device): # PyBuda configuration parameters compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/vision/perceiverio/test_perceiverio.py b/forge/test/models/pytorch/vision/perceiverio/test_perceiverio.py index f30ec30e7..05029b5e7 100644 --- a/forge/test/models/pytorch/vision/perceiverio/test_perceiverio.py +++ b/forge/test/models/pytorch/vision/perceiverio/test_perceiverio.py @@ -44,8 +44,9 @@ def get_sample_data(model_name): ] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_perceiverio_for_image_classification_pytorch(test_device, variant): # Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/rcnn/test_rcnn.py b/forge/test/models/pytorch/vision/rcnn/test_rcnn.py index c47154771..9613a6061 100644 --- a/forge/test/models/pytorch/vision/rcnn/test_rcnn.py +++ b/forge/test/models/pytorch/vision/rcnn/test_rcnn.py @@ -17,6 +17,7 @@ # Paper - https://arxiv.org/abs/1311.2524 # Repo - https://github.com/object-detection-algorithm/R-CNN @pytest.mark.nightly +@pytest.mark.model_analysis def test_rcnn_pytorch(test_device): # Load Alexnet Model diff --git a/forge/test/models/pytorch/vision/resnet/test_resnet.py b/forge/test/models/pytorch/vision/resnet/test_resnet.py index 0d0cadf99..a4b2cf68f 100644 --- a/forge/test/models/pytorch/vision/resnet/test_resnet.py +++ b/forge/test/models/pytorch/vision/resnet/test_resnet.py @@ -47,6 +47,7 @@ def generate_model_resnet_imgcls_hf_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnet(test_device): model, inputs, _ = generate_model_resnet_imgcls_hf_pytorch( @@ -83,6 +84,7 @@ def generate_model_resnet_imgcls_timm_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnet_timm(test_device): model, inputs, _ = generate_model_resnet_imgcls_timm_pytorch( "resnet50", diff --git a/forge/test/models/pytorch/vision/resnext/test_resnext.py b/forge/test/models/pytorch/vision/resnext/test_resnext.py index 79d50f147..13b13093f 100644 --- a/forge/test/models/pytorch/vision/resnext/test_resnext.py +++ b/forge/test/models/pytorch/vision/resnext/test_resnext.py @@ -14,6 +14,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnext_50_torchhub_pytorch(test_device): # STEP 1: Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object @@ -32,6 +33,7 @@ def test_resnext_50_torchhub_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnext_101_torchhub_pytorch(test_device): # STEP 1: Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object @@ -50,6 +52,7 @@ def test_resnext_101_torchhub_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnext_101_32x8d_fb_wsl_pytorch(test_device): # STEP 1: Set Forge configuration parameters @@ -70,6 +73,7 @@ def test_resnext_101_32x8d_fb_wsl_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnext_14_osmr_pytorch(test_device): # STEP 1: Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object @@ -89,6 +93,7 @@ def test_resnext_14_osmr_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnext_26_osmr_pytorch(test_device): # STEP 1: Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object @@ -107,6 +112,7 @@ def test_resnext_26_osmr_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnext_50_osmr_pytorch(test_device): # STEP 1: Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object @@ -125,6 +131,7 @@ def test_resnext_50_osmr_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_resnext_101_osmr_pytorch(test_device): # STEP 1: Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object diff --git a/forge/test/models/pytorch/vision/retinanet/test_retinanet.py b/forge/test/models/pytorch/vision/retinanet/test_retinanet.py index 9153cadca..18e74333d 100644 --- a/forge/test/models/pytorch/vision/retinanet/test_retinanet.py +++ b/forge/test/models/pytorch/vision/retinanet/test_retinanet.py @@ -22,8 +22,9 @@ ] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_retinanet(variant, test_device): # Set PyBuda configuration parameters diff --git a/forge/test/models/pytorch/vision/segformer/test_segformer.py b/forge/test/models/pytorch/vision/segformer/test_segformer.py index 792f1d7cf..45c2982d5 100644 --- a/forge/test/models/pytorch/vision/segformer/test_segformer.py +++ b/forge/test/models/pytorch/vision/segformer/test_segformer.py @@ -21,8 +21,9 @@ ] -@pytest.mark.parametrize("variant", variants_img_classification) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants_img_classification) def test_segformer_image_classification_pytorch(test_device, variant): # Set Forge configuration parameters @@ -56,8 +57,9 @@ def test_segformer_image_classification_pytorch(test_device, variant): ] -@pytest.mark.parametrize("variant", variants_semseg) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants_semseg) def test_segformer_semantic_segmentation_pytorch(test_device, variant): # Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/ssd300_resnet50/test_ssd300_resnet50.py b/forge/test/models/pytorch/vision/ssd300_resnet50/test_ssd300_resnet50.py index 5782f24c1..179f617b8 100644 --- a/forge/test/models/pytorch/vision/ssd300_resnet50/test_ssd300_resnet50.py +++ b/forge/test/models/pytorch/vision/ssd300_resnet50/test_ssd300_resnet50.py @@ -11,6 +11,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_pytorch_ssd300_resnet50(test_device): # STEP 1 : Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/swin/test_swin.py b/forge/test/models/pytorch/vision/swin/test_swin.py index 94f01ea9f..8f3dda218 100644 --- a/forge/test/models/pytorch/vision/swin/test_swin.py +++ b/forge/test/models/pytorch/vision/swin/test_swin.py @@ -13,6 +13,7 @@ @pytest.mark.nightly +@pytest.mark.model_analysis def test_swin_v1_tiny_4_224_hf_pytorch(test_device): # pytest.skip() # Working on it # STEP 1: Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/unet/test_unet.py b/forge/test/models/pytorch/vision/unet/test_unet.py index 42eb15c3d..27a999670 100644 --- a/forge/test/models/pytorch/vision/unet/test_unet.py +++ b/forge/test/models/pytorch/vision/unet/test_unet.py @@ -31,6 +31,7 @@ def generate_model_unet_imgseg_osmr_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_unet_osmr_cityscape_pytorch(test_device): model, inputs, _ = generate_model_unet_imgseg_osmr_pytorch( "unet_cityscapes", @@ -112,6 +113,7 @@ def generate_model_unet_imgseg_smp_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_unet_qubvel_pytorch(test_device): model, inputs, _ = generate_model_unet_imgseg_smp_pytorch( None, @@ -159,6 +161,7 @@ def generate_model_unet_imgseg_torchhub_pytorch(variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_unet_torchhub_pytorch(test_device): model, inputs, _ = generate_model_unet_imgseg_torchhub_pytorch( "unet", diff --git a/forge/test/models/pytorch/vision/vgg/test_vgg.py b/forge/test/models/pytorch/vision/vgg/test_vgg.py index 1259c24de..730ffd008 100644 --- a/forge/test/models/pytorch/vision/vgg/test_vgg.py +++ b/forge/test/models/pytorch/vision/vgg/test_vgg.py @@ -23,8 +23,9 @@ variants = ["vgg11", "vgg13", "vgg16", "vgg19", "bn_vgg19", "bn_vgg19b"] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_vgg_osmr_pytorch(variant, test_device): # STEP 1: Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() # load global compiler config object @@ -57,6 +58,7 @@ def test_vgg_osmr_pytorch(variant, test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_vgg_19_hf_pytorch(test_device): # STEP 1: Set Forge configuration parameters @@ -116,6 +118,7 @@ def preprocess_timm_model(model_name): @pytest.mark.nightly +@pytest.mark.model_analysis def test_vgg_bn19_timm_pytorch(test_device): torch.multiprocessing.set_sharing_strategy("file_system") model_name = "vgg19_bn" @@ -129,6 +132,7 @@ def test_vgg_bn19_timm_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.model_analysis def test_vgg_bn19_torchhub_pytorch(test_device): # STEP 1: Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/vit/test_vit.py b/forge/test/models/pytorch/vision/vit/test_vit.py index 4e737556d..8360774f1 100644 --- a/forge/test/models/pytorch/vision/vit/test_vit.py +++ b/forge/test/models/pytorch/vision/vit/test_vit.py @@ -34,8 +34,9 @@ def generate_model_vit_imgcls_hf_pytorch(test_device, variant): variants = ["google/vit-base-patch16-224", "google/vit-large-patch16-224"] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_vit_classify_224_hf_pytorch(variant, test_device): model, inputs, _ = generate_model_vit_imgcls_hf_pytorch( test_device, diff --git a/forge/test/models/pytorch/vision/vovnet/test_vovnet.py b/forge/test/models/pytorch/vision/vovnet/test_vovnet.py index 95f58a524..bed471652 100644 --- a/forge/test/models/pytorch/vision/vovnet/test_vovnet.py +++ b/forge/test/models/pytorch/vision/vovnet/test_vovnet.py @@ -27,8 +27,9 @@ def generate_model_vovnet_imgcls_osmr_pytorch(test_device, variant): varaints = ["vovnet27s", "vovnet39", "vovnet57"] -@pytest.mark.parametrize("variant", varaints, ids=varaints) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", varaints, ids=varaints) def test_vovnet_osmr_pytorch(variant, test_device): model, inputs, _ = generate_model_vovnet_imgcls_osmr_pytorch( test_device, @@ -47,8 +48,9 @@ def generate_model_vovnet39_imgcls_stigma_pytorch(test_device, variant): return model, [image_tensor], {} -@pytest.mark.parametrize("enable_default_dram_parameters", [True, False]) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("enable_default_dram_parameters", [True, False]) def test_vovnet_v1_39_stigma_pytorch(test_device, enable_default_dram_parameters): model, inputs, _ = generate_model_vovnet39_imgcls_stigma_pytorch( test_device, @@ -70,6 +72,7 @@ def generate_model_vovnet57_imgcls_stigma_pytorch(test_device, variant): @pytest.mark.nightly +@pytest.mark.model_analysis def test_vovnet_v1_57_stigma_pytorch(test_device): model, inputs, _ = generate_model_vovnet57_imgcls_stigma_pytorch( test_device, @@ -90,8 +93,9 @@ def generate_model_vovnet_imgcls_timm_pytorch(test_device, variant): variants = ["ese_vovnet19b_dw", "ese_vovnet39b", "ese_vovnet99b"] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_vovnet_timm_pytorch(variant, test_device): model, inputs, _ = generate_model_vovnet_imgcls_timm_pytorch( test_device, diff --git a/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py b/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py index 1c3b3a714..f6815effa 100644 --- a/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py +++ b/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py @@ -45,8 +45,9 @@ def generate_model_wideresnet_imgcls_pytorch(test_device, variant): variants = ["wide_resnet50_2", "wide_resnet101_2"] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_wideresnet_pytorch(variant, test_device): (model, inputs,) = generate_model_wideresnet_imgcls_pytorch( test_device, @@ -81,8 +82,9 @@ def generate_model_wideresnet_imgcls_timm(test_device, variant): variants = ["wide_resnet50_2", "wide_resnet101_2"] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_wideresnet_timm(variant, test_device): (model, inputs,) = generate_model_wideresnet_imgcls_timm( test_device, diff --git a/forge/test/models/pytorch/vision/xception/test_xception.py b/forge/test/models/pytorch/vision/xception/test_xception.py index fca12b1c3..62408acee 100644 --- a/forge/test/models/pytorch/vision/xception/test_xception.py +++ b/forge/test/models/pytorch/vision/xception/test_xception.py @@ -38,8 +38,9 @@ def generate_model_xception_imgcls_timm(test_device, variant): variants = ["xception", "xception41", "xception65", "xception71"] -@pytest.mark.parametrize("variant", variants, ids=variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants, ids=variants) def test_xception_timm(variant, test_device): (model, inputs,) = generate_model_xception_imgcls_timm( diff --git a/forge/test/models/pytorch/vision/yolo/test_yolo_v5.py b/forge/test/models/pytorch/vision/yolo/test_yolo_v5.py index 123f68dce..a7ac40320 100644 --- a/forge/test/models/pytorch/vision/yolo/test_yolo_v5.py +++ b/forge/test/models/pytorch/vision/yolo/test_yolo_v5.py @@ -24,8 +24,9 @@ def generate_model_yoloV5I320_imgcls_torchhub_pytorch(test_device, variant, size size = ["n", "s", "m", "l", "x"] -@pytest.mark.parametrize("size", size, ids=["yolov5" + s for s in size]) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("size", size, ids=["yolov5" + s for s in size]) def test_yolov5_320x320(test_device, size): model, inputs, _ = generate_model_yoloV5I320_imgcls_torchhub_pytorch( test_device, @@ -53,8 +54,9 @@ def generate_model_yoloV5I640_imgcls_torchhub_pytorch(test_device, variant, size size = ["n", "s", "m", "l", "x"] -@pytest.mark.parametrize("size", size, ids=["yolov5" + s for s in size]) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("size", size, ids=["yolov5" + s for s in size]) def test_yolov5_640x640(test_device, size): model, inputs, _ = generate_model_yoloV5I640_imgcls_torchhub_pytorch( @@ -78,8 +80,9 @@ def generate_model_yoloV5I480_imgcls_torchhub_pytorch(test_device, variant, size return model, [input_tensor], {} -@pytest.mark.parametrize("size", size, ids=["yolov5" + s for s in size]) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("size", size, ids=["yolov5" + s for s in size]) def test_yolov5_480x480(test_device, size): model, inputs, _ = generate_model_yoloV5I480_imgcls_torchhub_pytorch( @@ -93,6 +96,7 @@ def test_yolov5_480x480(test_device, size): @pytest.mark.nightly +@pytest.mark.model_analysis def test_yolov5_1280x1280(test_device): compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/vision/yolo/test_yolo_v6.py b/forge/test/models/pytorch/vision/yolo/test_yolo_v6.py index 4dfba9a3a..9cd47f45a 100644 --- a/forge/test/models/pytorch/vision/yolo/test_yolo_v6.py +++ b/forge/test/models/pytorch/vision/yolo/test_yolo_v6.py @@ -13,8 +13,9 @@ variants = ["yolov6n", "yolov6s", "yolov6m", "yolov6l"] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_yolo_v6_pytorch(variant, test_device): # STEP 1 : Set Forge configuration parameters diff --git a/forge/test/models/pytorch/vision/yolo/test_yolox.py b/forge/test/models/pytorch/vision/yolo/test_yolox.py index 8b912bc84..90064150b 100644 --- a/forge/test/models/pytorch/vision/yolo/test_yolox.py +++ b/forge/test/models/pytorch/vision/yolo/test_yolox.py @@ -33,8 +33,9 @@ variants = ["yolox_nano", "yolox_tiny", "yolox_s", "yolox_m", "yolox_l", "yolox_darknet", "yolox_x"] -@pytest.mark.parametrize("variant", variants) @pytest.mark.nightly +@pytest.mark.model_analysis +@pytest.mark.parametrize("variant", variants) def test_yolox_pytorch(variant, test_device): # Set PyBuda configuration parameters diff --git a/scripts/model_analysis.py b/scripts/model_analysis.py index b0136d25c..1c2c94c39 100644 --- a/scripts/model_analysis.py +++ b/scripts/model_analysis.py @@ -542,53 +542,56 @@ def dump_logs(log_file_dir_path: str, log_file_name: str, content: str): logger.info(f"Dumped test logs in {log_file}") -def collect_all_pytests(root_dir_path): +def collect_all_model_analysis_test(directory_or_file_path, output_directory_path): - assert check_path(root_dir_path), f"The directory path for collecting pytest {root_dir_path} doesn't exists" + assert check_path( + directory_or_file_path + ), f"The directory path for collecting test {directory_or_file_path} doesn't exists" - logger.info(f"Collecting all pytests in {root_dir_path}") + logger.info(f"Collecting all test that has model_analysis marker in {directory_or_file_path}") + collected_test_outputs = "" try: - res = subprocess.check_output(["pytest", root_dir_path, "--setup-plan"], stderr=subprocess.STDOUT).decode( - "utf-8" + result = subprocess.run( + ["pytest", directory_or_file_path, "-m", "model_analysis_new", "--collect-only"], + capture_output=True, + text=True, + check=True, ) - except subprocess.CalledProcessError as e: - output = e.output.decode("utf-8") - logger.error(f"[Error!] output = {output}") - return [] - test_list = [] - lines = res.split("\n") - for line in lines: - if "warnings summary" in line or "slowest durations" in line: - break + collected_test_outputs += "STDOUT:\n" + collected_test_outputs += result.stdout + collected_test_outputs += "STDERR:\n" + collected_test_outputs += result.stderr - if line and line.startswith(" " + root_dir_path) and "::" in line and "training" not in line: - line = line.strip() - line = line.split(" (fixtures used:")[0] if " (fixtures used:" in line else line - if "Grayskull" not in line and "Wormhole_B0" not in line: - test_list.append(line) + except subprocess.CalledProcessError as e: + collected_test_outputs += e.output - return test_list + dump_logs(output_directory_path, "collected_tests.txt", collected_test_outputs) + test_list = [] + with open(os.path.join(output_directory_path, "collected_tests.txt"), "r") as collected_test_file: + lines = collected_test_file.readlines() + test_lines = False + for line in lines: + if "Automatic Model Analysis Collected tests:" in line: + test_lines = True + elif "Automatic Model Analysis Collected test count:" in line: + test_lines = False + break + elif test_lines: + test_list.append(str(line).replace("\n", "")) -def generate_and_export_unique_ops_tests(pytest_directory_path, model_file_path, unique_ops_output_directory_path): + return test_list - # If model_file_path is specified, collect all the tests in the model_file_path parent directory path - # and in the test_list will only include the tests matching with model_file_path, - # otherwise collect all the tests in the pytest_directory_path specified by the user - if model_file_path: - model_file_path_list = model_file_path.split("/")[:-1] - tests_directory_path = "/".join(model_file_path_list) - else: - tests_directory_path = pytest_directory_path - test_list = collect_all_pytests(tests_directory_path) +def generate_and_export_unique_ops_tests(test_directory_or_file_path, unique_ops_output_directory_path): - if model_file_path: - test_list = [test for test in test_list if test.split("::")[0] == model_file_path] + test_list = collect_all_model_analysis_test(test_directory_or_file_path, unique_ops_output_directory_path) - assert test_list != [], f"No tests found in the {tests_directory_path} path" + assert ( + test_list != [] + ), f"No tests found in the {test_directory_or_file_path} path with model_analysis pytest marker" # Create a dictonary contains model_name as key and model tests(i.e include variant, task) as values model_name_to_tests = {} @@ -599,6 +602,11 @@ def generate_and_export_unique_ops_tests(pytest_directory_path, model_file_path, else: model_name_to_tests[model_name].append(test) + for model_name, tests in model_name_to_tests.items(): + print(f"{model_name}:") + for test in tests: + print(f"\t\t\t{test}") + # Generate unique op test for the all collected test and save the models unique ops test information in the unique_ops_output_directory_path model_output_dir_paths = [] for model_name, tests in model_name_to_tests.items(): @@ -941,21 +949,16 @@ def run_model_unique_op_tests_and_generate_markdowns( def main(): parser = argparse.ArgumentParser( - description="""Generate unique ops test for the models present in the pytest_directory_path or model_file_path + description="""Generate unique ops test for the models present in the test_directory_or_file_path specified by the user and run the unique ops test and generate markdown files, the root markdown file contains model name, variant name, framework and compiler components supported rate and sub-markdown file contains model variant unique op tests info""" ) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--pytest_directory_path", - type=str, - help="Specify the directory path containing models to test.", - ) - group.add_argument( - "--model_file_path", + parser.add_argument( + "--test_directory_or_file_path", type=str, - help="Specify the model file path to generate unique op tests and markdown file.", + default=os.path.join(os.getcwd(), "forge/test"), + help="Specify the directory or file path containing models test with model_analysis pytest marker", ) parser.add_argument( "--dump_failure_logs", @@ -964,29 +967,21 @@ def main(): ) parser.add_argument( "--markdown_directory_path", - default=os.path.join(os.getcwd(), "new_models_analysis_docs"), + default=os.path.join(os.getcwd(), "models_analysis_docs"), required=False, help="Specify the directory path for saving models information as markdowns file", ) parser.add_argument( "--unique_ops_output_directory_path", - default=os.path.join(os.getcwd(), "new_unique_ops"), + default=os.path.join(os.getcwd(), "unique_ops"), required=False, help="Specify the output directory path for saving models unique op tests outputs(i.e failure logs, xlsx file)", ) args = parser.parse_args() - if args.pytest_directory_path is not None: - assert check_path( - args.pytest_directory_path - ), f"Specified pytest directory path {args.pytest_directory_path} doesn't exists" - else: - assert check_path(args.model_file_path), f"Specified model file path {args.model_file_path} doesn't exists" - model_output_dir_paths = generate_and_export_unique_ops_tests( - pytest_directory_path=args.pytest_directory_path, - model_file_path=args.model_file_path, + test_directory_or_file_path=args.test_directory_or_file_path, unique_ops_output_directory_path=args.unique_ops_output_directory_path, )