chore: bringing Master in to Theseus

.dockerignore

@@ -103,6 +103,7 @@ common/test/data/badges/*.png
 ### Static assets pipeline artifacts
 **/*.scssc
 lms/static/css/
+!lms/static/css/vendor
 lms/static/certificates/css/
 cms/static/css/
 common/static/common/js/vendor/

.github/workflows/compile-python-requirements.yml

@@ -0,0 +1,72 @@
+name: Recompile Python dependencies
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Target branch to create requirements PR against'
+        required: true
+        default: 'master'
+        type: string
+
+defaults:
+  run:
+    shell: bash # making this explicit opts into -e -o pipefail
+
+jobs:
+  recompile-python-dependencies:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Check out target branch
+        uses: actions/checkout@v3
+        with:
+          ref: "${{ inputs.branch }}"
+
+      - name: Set up Python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Run make compile-requirements
+        env:
+          PACKAGE: "${{ inputs.package }}"
+        run: |
+          make compile-requirements
+
+      - name: PR preflight
+        run: |
+          if git diff --exit-code; then
+              # Fail early (and avoid quiet failure of create-pull-request action)
+              echo "Error: No changes, so not creating PR." | tee -a "$GITHUB_STEP_SUMMARY"
+              exit 1
+          fi
+
+      - name: Make a PR
+        id: make-pr
+        uses: peter-evans/create-pull-request@v5
+        with:
+          branch: "${{ github.triggering_actor }}/compile-python-deps"
+          branch-suffix: short-commit-hash
+          add-paths: requirements
+          commit-message: |
+            feat: Recompile Python dependencies
+
+            Commit generated by workflow `${{ github.workflow_ref }}`
+          title: "chore: Recompile Python dependencies"
+          body: >-
+            PR generated by workflow `${{ github.workflow_ref }}`
+            on behalf of @${{ github.triggering_actor }}.
+          assignees: "${{ github.triggering_actor }}"
+          reviewers: "${{ github.triggering_actor }}"
+
+      - name: Job summary
+        env:
+          PR_URL: "${{ steps.make-pr.outputs.pull-request-url }}"
+        run: |
+          if [[ -z "$PR_URL" ]]; then
+              echo "PR not created; see log for more information" | tee -a "$GITHUB_STEP_SUMMARY"
+              exit 1
+          else
+              echo "PR created or updated: $PR_URL" | tee -a "$GITHUB_STEP_SUMMARY"
+          fi

.github/workflows/publish-ci-docker-image.yml

@@ -14,7 +14,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_SECRET_ACCESS_KEY }}

.github/workflows/semgrep.yml

@@ -0,0 +1,45 @@
+# Finds code problems by structural pattern matching.
+#
+# New rules can be added to test_root/semgrep/ and they should be picked up
+# automatically. See https://semgrep.dev/docs/ for documentation.
+
+name: Semgrep code quality
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  run_semgrep:
+    name: Semgrep analysis
+    runs-on: "${{ matrix.os }}"
+    strategy:
+      matrix:
+        os: [ "ubuntu-20.04" ]
+        python-version: [ "3.8" ]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ matrix.python-version }}"
+
+      - name: Install semgrep
+        run: |
+          make pre-requirements
+          pip-sync requirements/edx/semgrep.txt
+
+      - name: Run semgrep
+        env:
+          # Peg this to some reasonable value so that semgrep's rewrapping
+          # of messages doesn't break up lines in an unpredictable manner:
+          # https://github.com/returntocorp/semgrep/issues/8608
+          COLUMNS: 80
+        run: |
+          semgrep scan --config test_root/semgrep/ --error --quiet \
+            -- lms cms common openedx

.github/workflows/upgrade-one-python-dependency.yml

@@ -2,32 +2,32 @@ name: Upgrade one Python dependency
 
 on:
   workflow_dispatch:
-     inputs:
-       branch:
-         description: 'Target branch to create requirements PR against'
-         required: true
-         default: 'master'
-         type: string
-       package:
-         description: 'Name of package to upgrade'
-         required: true
-         type: string
-       version:
-          description: 'Version number to upgrade to in constraints.txt (only needed if pinned)'
-          default: ''
-          type: string
-       change_desc:
-         description: |
-           Description of change, for commit message and PR. (What does the new version add or fix?)
-         default: ''
-         type: string
+    inputs:
+      branch:
+        description: 'Target branch to create requirements PR against'
+        required: true
+        default: 'master'
+        type: string
+      package:
+        description: 'Name of package to upgrade'
+        required: true
+        type: string
+      version:
+        description: 'Version number to upgrade to in constraints.txt (only needed if pinned)'
+        default: ''
+        type: string
+      change_desc:
+        description: |
+          Description of change, for commit message and PR. (What does the new version add or fix?)
+        default: ''
+        type: string
 
 defaults:
   run:
     shell: bash # making this explicit opts into -e -o pipefail
 
 jobs:
-  upgrade-one-python-dependency-workflow:
+  upgrade-one-python-dependency:
     runs-on: ubuntu-20.04
 
     steps:
@@ -92,8 +92,9 @@ jobs:
 
             ${{ env.body_prefix }}Commit generated by workflow `${{ github.workflow_ref }}`
           title: "feat: Upgrade Python dependency ${{ inputs.package }}"
-          body: |
-            ${{ env.body_prefix }}PR generated by workflow `${{ github.workflow_ref }}` on behalf of @${{ github.triggering_actor }}.
+          body: >-
+            ${{ env.body_prefix }}PR generated by workflow `${{ github.workflow_ref }}`
+            on behalf of @${{ github.triggering_actor }}.
           assignees: "${{ github.triggering_actor }}"
           reviewers: "${{ github.triggering_actor }}"
 

.readthedocs.yaml

@@ -6,7 +6,7 @@ build:
     python: "3.8"
 
 sphinx:
-  configuration: source/conf.py
+  configuration: docs/conf.py
 
 python:
   install:

Dockerfile

@@ -91,9 +91,6 @@ RUN apt-get update && \
     apt-get -y install --no-install-recommends \
         curl \
         libssl-dev \
-        libxml2-dev \
-        libxmlsec1-dev \
-        libxslt1-dev \
         libffi-dev \
         libfreetype6-dev \
         libgeos-dev \

Makefile

@@ -102,6 +102,7 @@ REQ_FILES = \
 	requirements/edx/testing \
 	requirements/edx/development \
 	requirements/edx/assets \
+	requirements/edx/semgrep \
 	scripts/xblock/requirements
 
 define COMMON_CONSTRAINTS_TEMP_COMMENT
@@ -111,7 +112,7 @@ endef
 COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
 .PHONY: $(COMMON_CONSTRAINTS_TXT)
 $(COMMON_CONSTRAINTS_TXT):
-	wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)"
+	wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
 	printf "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
 
 compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade

README.rst

@@ -25,6 +25,11 @@ platform.  Functionally, the edx-platform repository provides two services:
 * CMS (Content Management Service), which powers Open edX Studio, the platform's learning content authoring environment; and
 * LMS (Learning Management Service), which delivers learning content.
 
+Documentation
+*************
+
+Documentation can be found at https://docs.openedx.org/projects/edx-platform.
+
 Getting Started
 ***************
 
@@ -81,11 +86,6 @@ and other rich community resources.
 
 .. _Open edX site: https://openedx.org
 
-Documentation
-*************
-
-Documentation can be found at https://docs.openedx.org.
-
 
 Getting Help
 ************

cms/djangoapps/contentstore/asset_storage_handlers.py

@@ -88,6 +88,51 @@ def handle_assets(request, course_key_string=None, asset_key_string=None):
     return HttpResponseNotFound()
 
 
+def get_asset_usage_path(request, course_key, asset_key_string):
+    """
+    Get a list of units with ancestors that use given asset.
+    """
+    course_key = CourseKey.from_string(course_key)
+    if not has_course_author_access(request.user, course_key):
+        raise PermissionDenied()
+    asset_location = AssetKey.from_string(asset_key_string) if asset_key_string else None
+    store = modulestore()
+    usage_locations = []
+    static_path = StaticContent.get_static_path_from_location(asset_location)
+    verticals = store.get_items(
+        course_key,
+        qualifiers={
+            'category': 'vertical'
+        },
+    )
+    blocks = []
+
+    for vertical in verticals:
+        blocks.extend(vertical.get_children())
+
+    for block in blocks:
+        is_video_block = getattr(block, 'category', '') == 'video'
+        if is_video_block:
+            handout = getattr(block, 'handout', '')
+            if handout and str(asset_location) in handout:
+                unit = block.get_parent()
+                subsection = unit.get_parent()
+                subsection_display_name = getattr(subsection, 'display_name', '')
+                unit_display_name = getattr(unit, 'display_name', '')
+                xblock_display_name = getattr(block, 'display_name', '')
+                usage_locations.append(f'{subsection_display_name} - {unit_display_name} / {xblock_display_name}')
+        else:
+            data = getattr(block, 'data', '')
+            if static_path in data or str(asset_location) in data:
+                unit = block.get_parent()
+                subsection = unit.get_parent()
+                subsection_display_name = getattr(subsection, 'display_name', '')
+                unit_display_name = getattr(unit, 'display_name', '')
+                xblock_display_name = getattr(block, 'display_name', '')
+                usage_locations.append(f'{subsection_display_name} - {unit_display_name} / {xblock_display_name}')
+    return JsonResponse({'usage_locations': usage_locations})
+
+
 def _get_response_format(request):
     return request.GET.get('format') or request.POST.get('format') or 'html'
 

cms/djangoapps/contentstore/helpers.py

@@ -147,7 +147,7 @@ def xblock_type_display_name(xblock, default_display_name=None):
     elif category == 'vertical':
         return _('Unit')
     elif category == 'problem':
-        # The problem XBlock's display_name.default is not helpful ("Blank Advanced Problem") but changing it could have
+        # The problem XBlock's display_name.default is not helpful ("Blank Problem") but changing it could have
         # too many ripple effects in other places, so we have a special case for capa problems here.
         # Note: With a ProblemBlock instance, we could actually check block.problem_types to give a more specific
         # description like "Multiple Choice Problem", but that won't work if our 'block' argument is just the block_type

cms/djangoapps/contentstore/management/commands/tests/test_cleanup_assets.py

@@ -4,6 +4,7 @@
 """
 
 
+from unittest import skip
 from django.conf import settings
 from django.core.management import call_command
 from opaque_keys.edx.keys import CourseKey
@@ -20,6 +21,9 @@
 TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
 
 
+@skip("OldMongo Deprecation")
+# This test worked only for Old Mongo
+# Can later be converted to work with Split
 class ExportAllCourses(ModuleStoreTestCase):
     """
     Tests assets cleanup for all courses.