From 86dd8fd389aeeced4d233fe714522f15108c024b Mon Sep 17 00:00:00 2001 From: pchandrasekaran Date: Mon, 2 Dec 2024 09:25:16 +0000 Subject: [PATCH] Add Weekly job for model analysis --- .github/workflows/build-and-test.yml | 80 +++++++------- .github/workflows/model-analysis-weekly.yml | 59 ++++++++++ forge/test/conftest.py | 21 ++++ .../models/pytorch/text/bart/test_bart.py | 1 + .../text/distilbert/test_distilbert.py | 4 + .../vision/autoencoder/test_autoencoder.py | 2 + .../models/pytorch/vision/fpn/test_fpn.py | 1 + scripts/model_analysis.py | 103 +++++++++--------- 8 files changed, 177 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/model-analysis-weekly.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 323eafe77..6ac7d24bc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -146,43 +146,43 @@ jobs: source env/activate cmake --build ${{ steps.strings.outputs.build-output-dir }} -- run_unit_tests - - name: Run Test - shell: bash - run: | - source env/activate - apt install -y libgl1-mesa-glx - set -o pipefail # Ensures that the exit code reflects the first command that fails - pip install pytest-split - pytest -m push --splits 2 \ - --group ${{ matrix.test_group_id }} \ - --splitting-algorithm least_duration \ - -m "${{ inputs.test_mark }}" \ - --junit-xml=${{ steps.strings.outputs.test_report_path }} \ - 2>&1 | tee pytest.log - - - name: Upload Test Log - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: test-log-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }} - path: pytest.log - - - name: Run Perf Benchmark - shell: bash - run: | - source env/activate - python forge/test/benchmark/benchmark.py -m mnist_linear -bs 1 -o forge-benchmark-e2e-mnist.json - - - name: Upload Test Report - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: test-reports-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }} - path: ${{ steps.strings.outputs.test_report_path }} - - - name: Show Test Report - uses: mikepenz/action-junit-report@v4 - if: success() || failure() - with: - report_paths: ${{ steps.strings.outputs.test_report_path }} - check_name: TT-Forge-FE Tests + # - name: Run Test + # shell: bash + # run: | + # source env/activate + # apt install -y libgl1-mesa-glx + # set -o pipefail # Ensures that the exit code reflects the first command that fails + # pip install pytest-split + # pytest -m push --splits 2 \ + # --group ${{ matrix.test_group_id }} \ + # --splitting-algorithm least_duration \ + # -m "${{ inputs.test_mark }}" \ + # --junit-xml=${{ steps.strings.outputs.test_report_path }} \ + # 2>&1 | tee pytest.log + + # - name: Upload Test Log + # uses: actions/upload-artifact@v4 + # if: success() || failure() + # with: + # name: test-log-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }} + # path: pytest.log + + # - name: Run Perf Benchmark + # shell: bash + # run: | + # source env/activate + # python forge/test/benchmark/benchmark.py -m mnist_linear -bs 1 -o forge-benchmark-e2e-mnist.json + + # - name: Upload Test Report + # uses: actions/upload-artifact@v4 + # if: success() || failure() + # with: + # name: test-reports-${{ matrix.build.runs-on }}-${{ matrix.test_group_id }} + # path: ${{ steps.strings.outputs.test_report_path }} + + # - name: Show Test Report + # uses: mikepenz/action-junit-report@v4 + # if: success() || failure() + # with: + # report_paths: ${{ steps.strings.outputs.test_report_path }} + # check_name: TT-Forge-FE Tests diff --git a/.github/workflows/model-analysis-weekly.yml b/.github/workflows/model-analysis-weekly.yml new file mode 100644 index 000000000..b4d3a8900 --- /dev/null +++ b/.github/workflows/model-analysis-weekly.yml @@ -0,0 +1,59 @@ +name: Model Analysis Weekly + +on: + workflow_dispatch: + # schedule: + # - cron: '0 23 * * 5' # 11:00 PM UTC Friday (12:00 AM Saturday Serbia) + push: + branches: ["pchandrasekaran/model_analysis_weekly_job"] + +jobs: + build-and-push-test: + uses: ./.github/workflows/build-and-test.yml + secrets: inherit + with: + test_mark: 'push' + + model-analysis: + needs: build-and-push-test + runs-on: ${{ needs.build-and-push-test.outputs.matrix.build.runs-on }} + steps: + - name: Run Model Analysis Script + shell: bash + run: | + source env/activate + python scripts/model_analysis.py \ + --test_directory_or_file_path forge/test/models/pytorch \ + --dump_failure_logs \ + --markdown_directory_path ./model_analysis_docs \ + --unique_ops_output_directory_path ./models_unique_ops_output \ + 2>&1 | tee model_analysis.log + + - name: Upload Model Analysis Script Logs + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: model-analysis-outputs + path: model_analysis.log + + - name: Upload Models Unique Ops test Failure Logs + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: unique-ops-logs + path: ./models_unique_ops_output + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + branch: model_analysis + committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + base: main + commit-message: "Update model analysis docs" + title: "Update model analysis docs" + body: "This PR will update model analysis docs" + labels: model_analysis + delete-branch: true + files: ./model_analysis_docs + token: ${{ secrets.GH_TOKEN }} diff --git a/forge/test/conftest.py b/forge/test/conftest.py index 509858d55..1e7c3a55d 100644 --- a/forge/test/conftest.py +++ b/forge/test/conftest.py @@ -448,3 +448,24 @@ def pytest_runtest_logreport(report): for key, default_value in environ_before_test.items(): if os.environ.get(key, "") != default_value: os.environ[key] = default_value + + +def pytest_collection_modifyitems(config, items): + + marker = config.getoption("-m") # Get the marker from the -m option + + if marker and marker == "automatic_model_analysis": # If a marker is specified + filtered_items = [item for item in items if marker in item.keywords] + print("Automatic Model Analysis Collected tests: ") + test_count = 0 + for item in items: + if marker in item.keywords: + test_file_path = item.location[0] + test_name = item.location[2] + print(f"{test_file_path}::{test_name}") + test_count += 1 + print(f"Automatic Model Analysis Collected test count: {test_count}") + if not filtered_items: # Warn if no tests match the marker + print(f"Warning: No tests found with marker '{marker}'.") + else: + print(items) diff --git a/forge/test/models/pytorch/text/bart/test_bart.py b/forge/test/models/pytorch/text/bart/test_bart.py index 301a20374..260a63f19 100644 --- a/forge/test/models/pytorch/text/bart/test_bart.py +++ b/forge/test/models/pytorch/text/bart/test_bart.py @@ -23,6 +23,7 @@ def forward(self, input_ids, attention_mask, decoder_input_ids): @pytest.mark.nightly +@pytest.mark.automatic_model_analysis def test_pt_bart_classifier(test_device): compiler_cfg = _get_global_compiler_config() compiler_cfg.compile_depth = CompileDepth.SPLIT_GRAPH diff --git a/forge/test/models/pytorch/text/distilbert/test_distilbert.py b/forge/test/models/pytorch/text/distilbert/test_distilbert.py index 80362774d..7885faee1 100644 --- a/forge/test/models/pytorch/text/distilbert/test_distilbert.py +++ b/forge/test/models/pytorch/text/distilbert/test_distilbert.py @@ -16,6 +16,7 @@ @pytest.mark.nightly +@pytest.mark.automatic_model_analysis @pytest.mark.parametrize("variant", variants, ids=variants) def test_distilbert_masked_lm_pytorch(variant, test_device): # Load DistilBert tokenizer and model from HuggingFace @@ -46,6 +47,7 @@ def test_distilbert_masked_lm_pytorch(variant, test_device): @pytest.mark.nightly +@pytest.mark.automatic_model_analysis def test_distilbert_question_answering_pytorch(test_device): # Load Bert tokenizer and model from HuggingFace model_ckpt = "distilbert-base-cased-distilled-squad" @@ -82,6 +84,7 @@ def test_distilbert_question_answering_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.automatic_model_analysis def test_distilbert_sequence_classification_pytorch(test_device): # Load DistilBert tokenizer and model from HuggingFace @@ -109,6 +112,7 @@ def test_distilbert_sequence_classification_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.automatic_model_analysis def test_distilbert_token_classification_pytorch(test_device): # Load DistilBERT tokenizer and model from HuggingFace model_ckpt = "Davlan/distilbert-base-multilingual-cased-ner-hrl" diff --git a/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py b/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py index 5766cbf78..22d697f48 100644 --- a/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py +++ b/forge/test/models/pytorch/vision/autoencoder/test_autoencoder.py @@ -13,6 +13,7 @@ @pytest.mark.nightly +@pytest.mark.automatic_model_analysis def test_conv_ae_pytorch(test_device): # Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() @@ -40,6 +41,7 @@ def test_conv_ae_pytorch(test_device): @pytest.mark.nightly +@pytest.mark.automatic_model_analysis def test_linear_ae_pytorch(test_device): # Set Forge configuration parameters compiler_cfg = forge.config._get_global_compiler_config() diff --git a/forge/test/models/pytorch/vision/fpn/test_fpn.py b/forge/test/models/pytorch/vision/fpn/test_fpn.py index 8571a3fe5..daa451bae 100644 --- a/forge/test/models/pytorch/vision/fpn/test_fpn.py +++ b/forge/test/models/pytorch/vision/fpn/test_fpn.py @@ -8,6 +8,7 @@ @pytest.mark.nightly +@pytest.mark.automatic_model_analysis def test_fpn_pytorch(test_device): compiler_cfg = forge.config._get_global_compiler_config() compiler_cfg.compile_depth = forge.CompileDepth.SPLIT_GRAPH diff --git a/scripts/model_analysis.py b/scripts/model_analysis.py index b0136d25c..1499d172c 100644 --- a/scripts/model_analysis.py +++ b/scripts/model_analysis.py @@ -542,53 +542,56 @@ def dump_logs(log_file_dir_path: str, log_file_name: str, content: str): logger.info(f"Dumped test logs in {log_file}") -def collect_all_pytests(root_dir_path): +def collect_all_model_analysis_test(directory_or_file_path, output_directory_path): - assert check_path(root_dir_path), f"The directory path for collecting pytest {root_dir_path} doesn't exists" + assert check_path( + directory_or_file_path + ), f"The directory path for collecting test {directory_or_file_path} doesn't exists" - logger.info(f"Collecting all pytests in {root_dir_path}") + logger.info(f"Collecting all test that has automatic_model_analysis marker in {directory_or_file_path}") + collected_test_outputs = "" try: - res = subprocess.check_output(["pytest", root_dir_path, "--setup-plan"], stderr=subprocess.STDOUT).decode( - "utf-8" + result = subprocess.run( + ["pytest", directory_or_file_path, "-m", "automatic_model_analysis", "--collect-only"], + capture_output=True, + text=True, + check=True, ) - except subprocess.CalledProcessError as e: - output = e.output.decode("utf-8") - logger.error(f"[Error!] output = {output}") - return [] - test_list = [] - lines = res.split("\n") - for line in lines: - if "warnings summary" in line or "slowest durations" in line: - break + collected_test_outputs += "STDOUT:\n" + collected_test_outputs += result.stdout + collected_test_outputs += "STDERR:\n" + collected_test_outputs += result.stderr - if line and line.startswith(" " + root_dir_path) and "::" in line and "training" not in line: - line = line.strip() - line = line.split(" (fixtures used:")[0] if " (fixtures used:" in line else line - if "Grayskull" not in line and "Wormhole_B0" not in line: - test_list.append(line) + except subprocess.CalledProcessError as e: + collected_test_outputs += e.output - return test_list + dump_logs(output_directory_path, "collected_tests.txt", collected_test_outputs) + test_list = [] + with open(os.path.join(output_directory_path, "collected_tests.txt"), "r") as collected_test_file: + lines = collected_test_file.readlines() + test_lines = False + for line in lines: + if "Automatic Model Analysis Collected tests:" in line: + test_lines = True + elif "Automatic Model Analysis Collected test count:" in line: + test_lines = False + break + elif test_lines: + test_list.append(str(line).replace("\n", "")) -def generate_and_export_unique_ops_tests(pytest_directory_path, model_file_path, unique_ops_output_directory_path): + return test_list - # If model_file_path is specified, collect all the tests in the model_file_path parent directory path - # and in the test_list will only include the tests matching with model_file_path, - # otherwise collect all the tests in the pytest_directory_path specified by the user - if model_file_path: - model_file_path_list = model_file_path.split("/")[:-1] - tests_directory_path = "/".join(model_file_path_list) - else: - tests_directory_path = pytest_directory_path - test_list = collect_all_pytests(tests_directory_path) +def generate_and_export_unique_ops_tests(test_directory_or_file_path, unique_ops_output_directory_path): - if model_file_path: - test_list = [test for test in test_list if test.split("::")[0] == model_file_path] + test_list = collect_all_model_analysis_test(test_directory_or_file_path, unique_ops_output_directory_path) - assert test_list != [], f"No tests found in the {tests_directory_path} path" + assert ( + test_list != [] + ), f"No tests found in the {test_directory_or_file_path} path with automatic_model_analysis pytest marker" # Create a dictonary contains model_name as key and model tests(i.e include variant, task) as values model_name_to_tests = {} @@ -599,6 +602,11 @@ def generate_and_export_unique_ops_tests(pytest_directory_path, model_file_path, else: model_name_to_tests[model_name].append(test) + for model_name, tests in model_name_to_tests.items(): + print(f"{model_name}:") + for test in tests: + print(f"\t\t\t{test}") + # Generate unique op test for the all collected test and save the models unique ops test information in the unique_ops_output_directory_path model_output_dir_paths = [] for model_name, tests in model_name_to_tests.items(): @@ -941,21 +949,16 @@ def run_model_unique_op_tests_and_generate_markdowns( def main(): parser = argparse.ArgumentParser( - description="""Generate unique ops test for the models present in the pytest_directory_path or model_file_path + description="""Generate unique ops test for the models present in the test_directory_or_file_path specified by the user and run the unique ops test and generate markdown files, the root markdown file contains model name, variant name, framework and compiler components supported rate and sub-markdown file contains model variant unique op tests info""" ) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--pytest_directory_path", - type=str, - help="Specify the directory path containing models to test.", - ) - group.add_argument( - "--model_file_path", + parser.add_argument( + "--test_directory_or_file_path", type=str, - help="Specify the model file path to generate unique op tests and markdown file.", + default=os.path.join(os.getcwd(), "forge/test"), + help="Specify the directory or file path containing models test with automatic_model_analysis pytest marker", ) parser.add_argument( "--dump_failure_logs", @@ -964,29 +967,21 @@ def main(): ) parser.add_argument( "--markdown_directory_path", - default=os.path.join(os.getcwd(), "new_models_analysis_docs"), + default=os.path.join(os.getcwd(), "models_analysis_docs"), required=False, help="Specify the directory path for saving models information as markdowns file", ) parser.add_argument( "--unique_ops_output_directory_path", - default=os.path.join(os.getcwd(), "new_unique_ops"), + default=os.path.join(os.getcwd(), "unique_ops"), required=False, help="Specify the output directory path for saving models unique op tests outputs(i.e failure logs, xlsx file)", ) args = parser.parse_args() - if args.pytest_directory_path is not None: - assert check_path( - args.pytest_directory_path - ), f"Specified pytest directory path {args.pytest_directory_path} doesn't exists" - else: - assert check_path(args.model_file_path), f"Specified model file path {args.model_file_path} doesn't exists" - model_output_dir_paths = generate_and_export_unique_ops_tests( - pytest_directory_path=args.pytest_directory_path, - model_file_path=args.model_file_path, + test_directory_or_file_path=args.test_directory_or_file_path, unique_ops_output_directory_path=args.unique_ops_output_directory_path, )