From 76a8aaf6cd3e21fed44fd409adcb47691ce07ccf Mon Sep 17 00:00:00 2001 From: Aarif Date: Thu, 9 Dec 2021 18:21:17 +0500 Subject: [PATCH] fix: updated the unit tests workflow to simplify job names and read test paths from a separate file (#29472) --- .github/workflows/migrations-check.yml | 2 +- .github/workflows/pylint-checks.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/unit-test-shards.json | 14 +++++++ .github/workflows/unit-tests.yml | 38 ++++++++++--------- .../workflows/upgrade-python-requirements.yml | 2 +- .github/workflows/verify-dunder-init.yml | 2 +- .../workflows/verify-gha-unit-tests-count.yml | 4 +- scripts/gha_unit_tests_collector.py | 19 +++++----- scripts/unit_test_shards_parser.py | 27 +++++++++++++ 10 files changed, 79 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/unit-test-shards.json create mode 100644 scripts/unit_test_shards_parser.py diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 29bf9983de27..9c363c021303 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v2 diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 167fc4b9633a..e643efd61e4f 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -31,7 +31,7 @@ jobs: name: pylint ${{ matrix.module-name }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Install Required System Packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index a157944dcbd4..ab171be8df65 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -19,7 +19,7 @@ jobs: steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: fetch-depth: 2 diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json new file mode 100644 index 000000000000..e0b0754d0b36 --- /dev/null +++ b/.github/workflows/unit-test-shards.json @@ -0,0 +1,14 @@ +{ + "lms-1": "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/", + "lms-2": "lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_goals/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/", + "lms-3": "lms/djangoapps/courseware/", + "lms-4": "lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/", + "lms-5": "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/", + "lms-6": "lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy/ lms/djangoapps/save_for_later/ lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/envs/ lms/lib/ lms/tests.py", + "openedx-1": "openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/coursegraph/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/", + "openedx-2": "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/self_paced/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/", + "cms-1": "cms/djangoapps/api/ cms/djangoapps/cms_user_tasks/ cms/djangoapps/course_creators/ cms/djangoapps/export_course_metadata/ cms/djangoapps/maintenance/ cms/djangoapps/models/ cms/djangoapps/pipeline_js/ cms/djangoapps/xblock_config/ cms/envs/ cms/lib/", + "cms-2": "cms/djangoapps/contentstore/", + "common-1": "common/djangoapps/", + "common-2": "common/lib/" +} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8ef4f5d9d77b..7e8709a62b7e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -13,34 +13,34 @@ jobs: matrix: python-version: ['3.8'] django-version: ["3.2"] - test_module: [ - "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/", - "lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_goals/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/", - "lms/djangoapps/courseware/", - "lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/", - "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/", - "lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy/ lms/djangoapps/save_for_later/ lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/envs/ lms/lib/ lms/tests.py", - "openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/coursegraph/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/", - "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/self_paced/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/", - "cms/djangoapps/api/ cms/djangoapps/cms_user_tasks/ cms/djangoapps/course_creators/ cms/djangoapps/export_course_metadata/ cms/djangoapps/maintenance/ cms/djangoapps/models/ cms/djangoapps/pipeline_js/ cms/djangoapps/xblock_config/ cms/envs/ cms/lib/", - "cms/djangoapps/contentstore/", - "common/djangoapps/", - "common/lib/", + shard_name: [ + "lms-1", + "lms-2", + "lms-3", + "lms-4", + "lms-5", + "lms-6", + "openedx-1", + "openedx-2", + "cms-1", + "cms-2", + "common-1", + "common-2", ] - # name: python ${{ matrix.python-version }},django ${{ matrix.django-version }} ${{ matrix.test_module }} + name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} steps: - name: sync directory owner run: sudo chown runner:runner -R .* - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: start mongodb service run: | sudo /etc/init.d/mongodb start - name: set top-level module name run: | - echo "module_name=$(echo '${{ matrix.test_module }}' | awk -F '/' '{print $1}')" >> $GITHUB_ENV + echo "module_name=$(echo '${{ matrix.shard_name }}' | awk -F '-' '{print $1}')" >> $GITHUB_ENV - name: set settings path run: | @@ -56,6 +56,10 @@ jobs: sudo pip install -r requirements/edx/testing.txt sudo pip install "django~=${{ matrix.django-version }}.0" + - name: get unit tests for shard + run: | + echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV + - name: run tests run: | - python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ matrix.test_module }} + python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index 870ab692402d..241ee56ebc9b 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -23,7 +23,7 @@ jobs: - name: setup target branch run: echo "target_branch=$(if ['${{ github.event.inputs.branch }}' = '']; then echo 'master'; else echo '${{ github.event.inputs.branch }}'; fi)" >> $GITHUB_ENV - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: ref: ${{ env.target_branch }} diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml index e8a18222ac08..aefc0f53b6f5 100644 --- a/.github/workflows/verify-dunder-init.yml +++ b/.github/workflows/verify-dunder-init.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Check out branch - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Ensure git is installed run: | diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml index d5217957c0c0..1aa1a8f84b87 100644 --- a/.github/workflows/verify-gha-unit-tests-count.yml +++ b/.github/workflows/verify-gha-unit-tests-count.yml @@ -13,7 +13,7 @@ jobs: - name: sync directory owner run: sudo chown runner:runner -R .* - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: install requirements run: | sudo pip install -r requirements/pip.txt @@ -53,5 +53,5 @@ jobs: - name: verify unit tests count if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }} run: | - echo "::error title='Unit test modules in unit-tests.yml workflow are outdated'::unit tests running in unit-tests workflow don't match the count for unit tests for entire edx-platform suite, please update the test_module under matrix to add any missing apps and match the count" + echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows to add any missing apps and match the count" exit 1 diff --git a/scripts/gha_unit_tests_collector.py b/scripts/gha_unit_tests_collector.py index 710e89d914aa..67366211edfa 100644 --- a/scripts/gha_unit_tests_collector.py +++ b/scripts/gha_unit_tests_collector.py @@ -2,24 +2,25 @@ import os import yaml import argparse +import json -def get_all_unit_test_modules(): - unit_tests_yml = f'{os.getcwd()}/.github/workflows/unit-tests.yml' - with open(unit_tests_yml) as file: - unit_test_workflow_yaml = yaml.safe_load(file) +def get_all_unit_test_shards(): + unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json' + with open(unit_tests_json) as file: + unit_test_workflow_shards = json.loads(file.read()) - return unit_test_workflow_yaml['jobs']['run-tests']['strategy']['matrix']['test_module'] + return unit_test_workflow_shards def get_modules_except_cms(): - all_unit_test_modules = get_all_unit_test_modules() - return [module for module in all_unit_test_modules if not module.startswith('cms')] + all_unit_test_shards = get_all_unit_test_shards() + return [paths for shard_name, paths in all_unit_test_shards.items() if not paths.startswith('cms')] def get_cms_modules(): - all_unit_test_modules = get_all_unit_test_modules() - return [module for module in all_unit_test_modules if module.startswith('cms')] + all_unit_test_shards = get_all_unit_test_shards() + return [paths for shard_name, paths in all_unit_test_shards.items() if paths.startswith('cms')] if __name__ == "__main__": diff --git a/scripts/unit_test_shards_parser.py b/scripts/unit_test_shards_parser.py new file mode 100644 index 000000000000..ce9b08074417 --- /dev/null +++ b/scripts/unit_test_shards_parser.py @@ -0,0 +1,27 @@ +import sys +import os +import argparse +import json + + +def get_test_paths_for_shard(shard_name): + unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json' + with open(unit_tests_json) as file: + unit_test_workflow_shards = json.loads(file.read()) + + if shard_name not in unit_test_workflow_shards: + sys.stdout.write("Error, invalid shard name provided. please provide a valid shard name as specified in unit-test-shards.json") + + return unit_test_workflow_shards.get(shard_name) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--shard-name", action="store", default="") + argument = parser.parse_args() + + if not argument.shard_name: + sys.stdout.write("Error, no shard name provided. please provide a valid shard name as specified in unit-test-shards.json") + + unit_test_paths = get_test_paths_for_shard(argument.shard_name) + sys.stdout.write(unit_test_paths)