Add Weekly job for model analysis

.github/workflows/build-and-test.yml

@@ -146,43 +146,43 @@ jobs:
         source env/activate
         cmake --build ${{ steps.strings.outputs.build-output-dir }} -- run_unit_tests
 
-    - name: Run Test
-      shell: bash
-      run: |
-        source env/activate
-        apt install -y libgl1-mesa-glx
-        set -o pipefail # Ensures that the exit code reflects the first command that fails
-        pip install pytest-split
-        pytest -m push --splits 2 \
-               --group ${{ matrix.test_group_id }} \
-               --splitting-algorithm least_duration \
-               -m "${{ inputs.test_mark }}" \
-               --junit-xml=${{ steps.strings.outputs.test_report_path }} \
-               2>&1 | tee pytest.log
-
-    - name: Upload Test Log
-      uses: actions/upload-artifact@v4
-      if: success() || failure()
-      with:
-        name: test-log-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }}
-        path: pytest.log
-
-    - name: Run Perf Benchmark
-      shell: bash
-      run: |
-        source env/activate
-        python forge/test/benchmark/benchmark.py -m mnist_linear -bs 1 -o forge-benchmark-e2e-mnist.json
-
-    - name: Upload Test Report
-      uses: actions/upload-artifact@v4
-      if: success() || failure()
-      with:
-        name: test-reports-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }}
-        path: ${{ steps.strings.outputs.test_report_path }}
-
-    - name: Show Test Report
-      uses: mikepenz/action-junit-report@v4
-      if: success() || failure()
-      with:
-        report_paths: ${{ steps.strings.outputs.test_report_path }}
-        check_name: TT-Forge-FE Tests
+    # - name: Run Test
+    #   shell: bash
+    #   run: |
+    #     source env/activate
+    #     apt install -y libgl1-mesa-glx
+    #     set -o pipefail # Ensures that the exit code reflects the first command that fails
+    #     pip install pytest-split
+    #     pytest -m push --splits 2 \
+    #            --group ${{ matrix.test_group_id }} \
+    #            --splitting-algorithm least_duration \
+    #            -m "${{ inputs.test_mark }}" \
+    #            --junit-xml=${{ steps.strings.outputs.test_report_path }} \
+    #            2>&1 | tee pytest.log
+
+    # - name: Upload Test Log
+    #   uses: actions/upload-artifact@v4
+    #   if: success() || failure()
+    #   with:
+    #     name: test-log-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }}
+    #     path: pytest.log
+
+    # - name: Run Perf Benchmark
+    #   shell: bash
+    #   run: |
+    #     source env/activate
+    #     python forge/test/benchmark/benchmark.py -m mnist_linear -bs 1 -o forge-benchmark-e2e-mnist.json
+
+    # - name: Upload Test Report
+    #   uses: actions/upload-artifact@v4
+    #   if: success() || failure()
+    #   with:
+    #     name: test-reports-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }}
+    #     path: ${{ steps.strings.outputs.test_report_path }}
+
+    # - name: Show Test Report
+    #   uses: mikepenz/action-junit-report@v4
+    #   if: success() || failure()
+    #   with:
+    #     report_paths: ${{ steps.strings.outputs.test_report_path }}
+    #     check_name: TT-Forge-FE Tests

.github/workflows/model-analysis-weekly.yml

@@ -0,0 +1,142 @@
+name: Model Analysis Weekly
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: '0 23 * * 5' # 11:00 PM UTC Friday (12:00 AM Saturday Serbia)
+  push:
+    branches: ["pchandrasekaran/model_analysis_weekly_job"]
+
+jobs:
+
+  build-image:
+    runs-on: builder
+    outputs:
+      docker-image: ${{ steps.build.outputs.docker-image }}
+    steps:
+      - name: Fix permissions
+        shell: bash
+        run: sudo chown ubuntu:ubuntu -R $(pwd)
+
+      - uses: actions/checkout@v4
+        with:
+            submodules: recursive
+            fetch-depth: 0 # Fetch all history and tags
+
+      # Clean everything from submodules (needed to avoid issues
+      # with cmake generated files leftover from previous builds)
+      - name: Cleanup submodules
+        run: |
+          git submodule foreach --recursive git clean -ffdx
+          git submodule foreach --recursive git reset --hard
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Docker images and output the image name
+        id: build
+        shell: bash
+        run: |
+          # Output the image name
+          set pipefail
+          .github/build-docker-images.sh | tee docker.log
+          DOCKER_CI_IMAGE=$(tail -n 1 docker.log)
+          echo "DOCKER_CI_IMAGE $DOCKER_CI_IMAGE"
+          echo "docker-image=$DOCKER_CI_IMAGE" >> "$GITHUB_OUTPUT"
+
+  model-analysis:
+
+    needs: build-image
+    runs-on: in-service
+
+    container:
+      image: ${{ needs.build-image.outputs.docker-image }}
+      options: --device /dev/tenstorrent/0
+      volumes:
+        - /dev/hugepages:/dev/hugepages
+        - /dev/hugepages-1G:/dev/hugepages-1G
+        - /etc/udev/rules.d:/etc/udev/rules.d
+        - /lib/modules:/lib/modules
+        - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
+
+    steps:
+
+      - name: Set reusable strings
+        id: strings
+        shell: bash
+        run: |
+          echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+          echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+
+      - name: Git safe dir
+        run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
+
+      - uses: actions/checkout@v4
+        with:
+            submodules: recursive
+            fetch-depth: 0 # Fetch all history and tags
+
+      # Clean everything from submodules (needed to avoid issues
+      # with cmake generated files leftover from previous builds)
+      - name: Cleanup submodules
+        run: |
+            git submodule foreach --recursive git clean -ffdx
+            git submodule foreach --recursive git reset --hard
+
+      - name: Build
+        shell: bash
+        run: |
+          source env/activate
+          cmake -G Ninja \
+          -B ${{ steps.strings.outputs.build-output-dir }} \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_C_COMPILER=clang \
+          -DCMAKE_CXX_COMPILER=clang++ \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+          cmake --build ${{ steps.strings.outputs.build-output-dir }}
+
+      - name: Run Model Analysis Script
+        shell: bash
+        run: |
+          source env/activate
+          apt install -y libgl1-mesa-glx
+          python scripts/model_analysis.py \
+            --test_directory_or_file_path forge/test/models/pytorch \
+            --dump_failure_logs \
+            --markdown_directory_path ./model_analysis_docs \
+            --unique_ops_output_directory_path ./models_unique_ops_output \
+            2>&1 | tee model_analysis.log
+
+      - name: Upload Model Analysis Script Logs
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: model-analysis-outputs
+          path: model_analysis.log
+
+      - name: Upload Models Unique Ops test Failure Logs
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: unique-ops-logs
+          path: ./models_unique_ops_output
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          branch: model_analysis
+          committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          base: main
+          commit-message: "Update model analysis docs"
+          title: "Update model analysis docs"
+          body: "This PR will update model analysis docs"
+          labels: model_analysis
+          delete-branch: true
+          files: ./model_analysis_docs
+          token: ${{ secrets.GH_TOKEN }}

forge/test/conftest.py

@@ -448,3 +448,24 @@ def pytest_runtest_logreport(report):
         for key, default_value in environ_before_test.items():
             if os.environ.get(key, "") != default_value:
                 os.environ[key] = default_value
+
+
+def pytest_collection_modifyitems(config, items):
+
+    marker = config.getoption("-m")  # Get the marker from the -m option
+
+    if marker and marker == "automatic_model_analysis":  # If a marker is specified
+        filtered_items = [item for item in items if marker in item.keywords]
+        print("Automatic Model Analysis Collected tests: ")
+        test_count = 0
+        for item in items:
+            if marker in item.keywords:
+                test_file_path = item.location[0]
+                test_name = item.location[2]
+                print(f"{test_file_path}::{test_name}")
+                test_count += 1
+        print(f"Automatic Model Analysis Collected test count: {test_count}")
+        if not filtered_items:  # Warn if no tests match the marker
+            print(f"Warning: No tests found with marker '{marker}'.")
+    else:
+        print(items)

forge/test/models/pytorch/text/bart/test_bart.py

@@ -23,6 +23,7 @@ def forward(self, input_ids, attention_mask, decoder_input_ids):
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 def test_pt_bart_classifier(test_device):
     compiler_cfg = _get_global_compiler_config()
     compiler_cfg.compile_depth = CompileDepth.SPLIT_GRAPH

forge/test/models/pytorch/text/distilbert/test_distilbert.py

@@ -16,6 +16,7 @@
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_distilbert_masked_lm_pytorch(variant, test_device):
     # Load DistilBert tokenizer and model from HuggingFace
@@ -46,6 +47,7 @@ def test_distilbert_masked_lm_pytorch(variant, test_device):
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 def test_distilbert_question_answering_pytorch(test_device):
     # Load Bert tokenizer and model from HuggingFace
     model_ckpt = "distilbert-base-cased-distilled-squad"
@@ -82,6 +84,7 @@ def test_distilbert_question_answering_pytorch(test_device):
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 def test_distilbert_sequence_classification_pytorch(test_device):
 
     # Load DistilBert tokenizer and model from HuggingFace
@@ -109,6 +112,7 @@ def test_distilbert_sequence_classification_pytorch(test_device):
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 def test_distilbert_token_classification_pytorch(test_device):
     # Load DistilBERT tokenizer and model from HuggingFace
     model_ckpt = "Davlan/distilbert-base-multilingual-cased-ner-hrl"

forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py

@@ -13,6 +13,7 @@
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 def test_conv_ae_pytorch(test_device):
     # Set Forge configuration parameters
     compiler_cfg = forge.config._get_global_compiler_config()
@@ -40,6 +41,7 @@ def test_conv_ae_pytorch(test_device):
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 def test_linear_ae_pytorch(test_device):
     # Set Forge configuration parameters
     compiler_cfg = forge.config._get_global_compiler_config()

forge/test/models/pytorch/vision/fpn/test_fpn.py

@@ -8,6 +8,7 @@
 
 
 @pytest.mark.nightly
+@pytest.mark.automatic_model_analysis
 def test_fpn_pytorch(test_device):
     compiler_cfg = forge.config._get_global_compiler_config()
     compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH