fix: updated the unit tests workflow to simplify job names and read t…

…est paths from a separate file (#29472)

.github/workflows/migrations-check.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Setup Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2

.github/workflows/pylint-checks.yml

@@ -31,7 +31,7 @@ jobs:
     name: pylint ${{ matrix.module-name }}
     steps:
 
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
 
     - name: Install Required System Packages
       run: sudo apt-get update && sudo apt-get install libxmlsec1-dev

.github/workflows/quality-checks.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
 
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         fetch-depth: 2
 

.github/workflows/unit-test-shards.json

@@ -0,0 +1,14 @@
+{
+  "lms-1": "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/",
+  "lms-2": "lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_goals/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/",
+  "lms-3": "lms/djangoapps/courseware/",
+  "lms-4": "lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/",
+  "lms-5": "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/",
+  "lms-6": "lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy/ lms/djangoapps/save_for_later/ lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/envs/ lms/lib/ lms/tests.py",
+  "openedx-1": "openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/coursegraph/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/",
+  "openedx-2": "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/self_paced/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/",
+  "cms-1": "cms/djangoapps/api/ cms/djangoapps/cms_user_tasks/ cms/djangoapps/course_creators/ cms/djangoapps/export_course_metadata/ cms/djangoapps/maintenance/ cms/djangoapps/models/ cms/djangoapps/pipeline_js/ cms/djangoapps/xblock_config/ cms/envs/ cms/lib/",
+  "cms-2": "cms/djangoapps/contentstore/",
+  "common-1": "common/djangoapps/",
+  "common-2": "common/lib/"
+}

.github/workflows/unit-tests.yml

@@ -13,34 +13,34 @@ jobs:
       matrix:
         python-version: ['3.8']
         django-version: ["3.2"]
-        test_module: [
-            "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/",
-            "lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_goals/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/",
-            "lms/djangoapps/courseware/",
-            "lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/",
-            "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/",
-            "lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy/ lms/djangoapps/save_for_later/ lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/envs/ lms/lib/ lms/tests.py",
-            "openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/coursegraph/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/",
-            "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/self_paced/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/",
-            "cms/djangoapps/api/ cms/djangoapps/cms_user_tasks/ cms/djangoapps/course_creators/ cms/djangoapps/export_course_metadata/ cms/djangoapps/maintenance/ cms/djangoapps/models/ cms/djangoapps/pipeline_js/ cms/djangoapps/xblock_config/ cms/envs/ cms/lib/",
-            "cms/djangoapps/contentstore/",
-            "common/djangoapps/",
-            "common/lib/",
+        shard_name: [
+            "lms-1",
+            "lms-2",
+            "lms-3",
+            "lms-4",
+            "lms-5",
+            "lms-6",
+            "openedx-1",
+            "openedx-2",
+            "cms-1",
+            "cms-2",
+            "common-1",
+            "common-2",
         ]
 
 
-    # name: python ${{ matrix.python-version }},django ${{ matrix.django-version }} ${{ matrix.test_module }}
+    name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }}
     steps:
       - name: sync directory owner
         run: sudo chown runner:runner -R .*
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: start mongodb service
         run: |
           sudo /etc/init.d/mongodb start
 
       - name: set top-level module name
         run: |
-          echo "module_name=$(echo '${{ matrix.test_module }}' | awk -F '/' '{print $1}')" >> $GITHUB_ENV
+          echo "module_name=$(echo '${{ matrix.shard_name }}' | awk -F '-' '{print $1}')" >> $GITHUB_ENV
 
       - name: set settings path
         run: |
@@ -56,6 +56,10 @@ jobs:
           sudo pip install -r requirements/edx/testing.txt
           sudo pip install "django~=${{ matrix.django-version }}.0"
 
+      - name: get unit tests for shard
+        run: |
+          echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV
+
       - name: run tests
         run: |
-          python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ matrix.test_module }}
+          python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }}

.github/workflows/upgrade-python-requirements.yml

@@ -23,7 +23,7 @@ jobs:
       - name: setup target branch
         run: echo "target_branch=$(if ['${{ github.event.inputs.branch }}' = '']; then echo 'master'; else echo '${{ github.event.inputs.branch }}'; fi)" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           ref: ${{ env.target_branch }}
 

.github/workflows/verify-dunder-init.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
 
     - name: Check out branch
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Ensure git is installed
       run: |

.github/workflows/verify-gha-unit-tests-count.yml

@@ -13,7 +13,7 @@ jobs:
       - name: sync directory owner
         run: sudo chown runner:runner -R .*
 
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: install requirements
         run: |
           sudo pip install -r requirements/pip.txt
@@ -53,5 +53,5 @@ jobs:
       - name: verify unit tests count
         if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }}
         run: |
-          echo "::error title='Unit test modules in unit-tests.yml workflow are outdated'::unit tests running in unit-tests workflow don't match the count for unit tests for entire edx-platform suite, please update the test_module under matrix to add any missing apps and match the count"
+          echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows to add any missing apps and match the count"
           exit 1

scripts/gha_unit_tests_collector.py

@@ -2,24 +2,25 @@
 import os
 import yaml
 import argparse
+import json
 
 
-def get_all_unit_test_modules():
-    unit_tests_yml = f'{os.getcwd()}/.github/workflows/unit-tests.yml'
-    with open(unit_tests_yml) as file:
-        unit_test_workflow_yaml = yaml.safe_load(file)
+def get_all_unit_test_shards():
+    unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json'
+    with open(unit_tests_json) as file:
+        unit_test_workflow_shards = json.loads(file.read())
 
-    return unit_test_workflow_yaml['jobs']['run-tests']['strategy']['matrix']['test_module']
+    return unit_test_workflow_shards
 
 
 def get_modules_except_cms():
-    all_unit_test_modules = get_all_unit_test_modules()
-    return [module for module in all_unit_test_modules if not module.startswith('cms')]
+    all_unit_test_shards = get_all_unit_test_shards()
+    return [paths for shard_name, paths in all_unit_test_shards.items() if not paths.startswith('cms')]
 
 
 def get_cms_modules():
-    all_unit_test_modules = get_all_unit_test_modules()
-    return [module for module in all_unit_test_modules if module.startswith('cms')]
+    all_unit_test_shards = get_all_unit_test_shards()
+    return [paths for shard_name, paths in all_unit_test_shards.items() if paths.startswith('cms')]
 
 
 if __name__ == "__main__":

scripts/unit_test_shards_parser.py

@@ -0,0 +1,27 @@
+import sys
+import os
+import argparse
+import json
+
+
+def get_test_paths_for_shard(shard_name):
+    unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json'
+    with open(unit_tests_json) as file:
+        unit_test_workflow_shards = json.loads(file.read())
+
+    if shard_name not in unit_test_workflow_shards:
+        sys.stdout.write("Error, invalid shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
+
+    return unit_test_workflow_shards.get(shard_name)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--shard-name", action="store", default="")
+    argument = parser.parse_args()
+
+    if not argument.shard_name:
+        sys.stdout.write("Error, no shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
+
+    unit_test_paths = get_test_paths_for_shard(argument.shard_name)
+    sys.stdout.write(unit_test_paths)