Revert "Paver deprecation - attempt #?"

docker/build/edxapp/ansible_overrides.yml

@@ -12,6 +12,7 @@ edxapp_npm_production: "no"
 
 EDXAPP_LMS_GUNICORN_EXTRA_CONF: 'reload = True'
 
+EDXAPP_NO_PREREQ_INSTALL: 0
 COMMON_SSH_PASSWORD_AUTH: "yes"
 EDXAPP_CMS_BASE: "edx.devstack.studio:18010"
 EDXAPP_OAUTH_ENFORCE_SECURE: false

playbooks/roles/aws_devstack/tasks/main.yml

@@ -37,7 +37,7 @@
       cd edx-repos/devstack/
       make dev.pull.large-and-slow
       make dev.up.lms
-      . /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm ci && npm run build && ./manage.py lms collectstatic --noinput && ./manage.py cms collectstatic --noinput && python manage.py lms migrate'
+      . /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm ci && paver update_assets && python manage.py lms migrate'
   async: 1800
   poll: 10
   become_user: ubuntu

playbooks/roles/edxapp/defaults/main.yml

@@ -487,6 +487,9 @@ EDXAPP_CMS_STATIC_URL_BASE: "{{ EDXAPP_STATIC_URL_BASE }}"
 # does not affect verified students
 EDXAPP_PAID_COURSE_REGISTRATION_CURRENCY: ['usd', '$']
 
+# Configure paver tasks in edx-platform to skip Python/Ruby/Node installation
+EDXAPP_NO_PREREQ_INSTALL: 1
+
 # whether to setup the python codejail or not
 EDXAPP_PYTHON_SANDBOX: true
 # this next setting, if true, turns on actual sandbox enforcement.  If not true,
@@ -1173,6 +1176,7 @@ edxapp_helper_scripts:
 
 edxapp_environment_default:
   LANG: "{{ EDXAPP_LANG }}"
+  NO_PREREQ_INSTALL: "{{ EDXAPP_NO_PREREQ_INSTALL }}"
   SKIP_WS_MIGRATIONS: 1
   PATH: "{{ edxapp_deploy_path }}"
   # the settings module for edxapp, DJANGO_SETTINGS_MODULE

playbooks/roles/edxapp/tasks/main.yml

@@ -49,6 +49,15 @@
     - devstack
     - devstack:install
 
+- name: add paver autocomplete to bashrc
+  lineinfile:
+    dest: /root/.bashrc
+    line: "source {{ edxapp_code_dir }}/scripts/paver_autocomplete.sh"
+  when: devstack is defined and devstack
+  tags:
+  - devstack
+  - devstack:install
+
 - name: create edxapp log dir
   file:
     path: "{{ edxapp_log_dir }}"

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -312,8 +312,8 @@
     - gather_static_assets
     - assets
 
-# Build static assets if possible
-- name: "gather static assets"
+# Gather assets using paver if possible
+- name: "gather static assets with paver and pull translations"
   command: "{{ COMMON_BIN_DIR }}/edxapp-update-assets"
   when: celery_worker is not defined and not devstack
   tags:

playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2

@@ -1,32 +1,13 @@
 {% include "edxapp_common.j2" %}
 
-export WEBPACK_CONFIG_PATH=webpack.prod.config.js
-export STATIC_ROOT_LMS=/edx/var/edxapp/staticfiles
-export STATIC_ROOT_CMS=/edx/var/edxapp/staticfiles/studio
-# This is necessary to properly build webworkers, which depends on this value being a dictionary.
-export JS_ENV_EXTRA_CONFIG={}
-
-
-# We need to make a call to a management command before running webpack because proctoring runs a webpack
-# webworker that only gets built if the proctoring djangoapp writes out a `workers.json`:
-# https://github.com/openedx/edx-proctoring/blob/73c7f55e2be91324fa07fec6e6ac0a667fdd8412/edx_proctoring/apps.py#L4
 {% if edxapp_staticfiles_storage_overrides %}
 {% for override in edxapp_staticfiles_storage_overrides %}
-export STATICFILES_STORAGE={{ override | quote }}
 sudo -E -H -u {{ edxapp_user }} \
-    env "PATH=$PATH" \
-    {{ edxapp_venv_bin }} python manage.py lms --settings=$EDX_PLATFORM_SETTINGS print_setting STATIC_ROOT WEBPACK_CONFIG_PATH \
-    && npm run webpack \
-    && npm run compile-sass -- --theme-dir /edx/var/edx-themes/edx-themes/edx-platform --theme-dir /edx/app/edxapp/edx-platform/themes \
-    && {{ edxapp_venv_bin }}/python manage.py lms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS \
-    && {{ edxapp_venv_bin }}/python manage.py cms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS
+    env "PATH=$PATH" "STATICFILES_STORAGE={{ override }}" \
+    {{ edxapp_venv_bin }}/paver update_assets --debug-collect --settings=$EDX_PLATFORM_SETTINGS
 {% endfor %}
 {% else %}
 sudo -E -H -u {{ edxapp_user }} \
     env "PATH=$PATH" \
-    {{ edxapp_venv_bin }} python manage.py lms --settings=$EDX_PLATFORM_SETTINGS print_setting STATIC_ROOT WEBPACK_CONFIG_PATH \
-    && npm run webpack \
-    && npm run compile-sass -- --theme-dir /edx/var/edx-themes/edx-themes/edx-platform --theme-dir /edx/app/edxapp/edx-platform/themes \
-    && {{ edxapp_venv_bin }}/python manage.py lms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS \
-    && {{ edxapp_venv_bin }}/python manage.py cms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS
+    {{ edxapp_venv_bin }}/paver update_assets --debug-collect --settings $EDX_PLATFORM_SETTINGS
 {% endif %}

playbooks/roles/test_build_server/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+#
+# edX Configuration
+#
+# github:     https://github.com/openedx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/openedx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role test_build_server
+#
+
+#
+# vars are namespace with the module name.
+#
+test_build_server_user: jenkins
+test_build_server_repo_path: /home/jenkins
+TEST_EDX_PLATFORM_VERSION: master

playbooks/roles/test_build_server/files/test-development-environment.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -e
+################################################################################
+# This executes a small subset of the edx-platform tests.  It is intended as
+# a means of testing newly provisioned AMIs for our jenkins workers.
+#
+# The two main things that happen here:
+#   1. The setup from edx-platform/scripts/all-tests.sh, the script that is
+#      run by the jenkins workers to kick off tests.
+#   2. The paver command for tests, coverage and quality reports are run.
+#      For the tests, it runs only a small number of test cases for each
+#      test suite.
+###############################################################################
+
+# Doing this rather than copying the file into the scripts folder so that
+# this file doesn't get cleaned out by the 'git clean' in all-tests.sh.
+cd edx-platform-clone
+
+# This will run all of the setup it usually runs, but none of the
+# tests because TEST_SUITE isn't defined.
+export PYTHON_VERSION=3.8
+source scripts/jenkins-common.sh
+
+case "$1" in
+    "unit")
+
+        # Now we can run a subset of the tests via paver.
+        # Run some of the common/lib unit tests
+        paver test_lib -t common/lib/xmodule/xmodule/tests/test_stringify.py
+
+        # Generate some coverage reports
+        # Since `TEST_SUITE` is not set, change the coverage file written by the
+        # previous test to a generic one.
+        cp reports/common_lib_xmodule.coverage reports/.coverage
+        paver coverage
+
+        # Run some of the djangoapp unit tests
+        paver test_system -t lms/djangoapps/courseware/tests/tests.py
+        paver test_system -t cms/djangoapps/course_creators/tests/test_views.py
+        ;;
+
+    "js")
+
+        # Run some of the javascript unit tests
+        paver test_js_run -s lms-coffee
+        ;;
+
+    "quality")
+        # Generate quality reports
+        paver run_quality
+        ;;
+
+    *)
+        echo "args required"
+        exit 1
+esac

playbooks/roles/test_build_server/meta/main.yml

@@ -0,0 +1,26 @@
+---
+#
+# edX Configuration
+#
+# github:     https://github.com/openedx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/openedx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role test_build_server
+#
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+
+#### INTENTIONALLY LEFT BLANK ####
+
+# Since this is a test role, it should not install anything extra onto the
+# target machine, thus altering the system under test. Be careful when
+# adding dependencies.

playbooks/roles/test_build_server/tasks/main.yml

@@ -0,0 +1,48 @@
+---
+#
+# edX Configuration
+#
+# github:     https://github.com/openedx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/openedx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role test_build_server
+#
+# Overview:
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+
+- name: Create clone of edx-platform
+  git:
+    repo: "https://github.com/openedx/edx-platform.git"
+    dest: "{{ test_build_server_repo_path }}/edx-platform-clone"
+    version: "{{ TEST_EDX_PLATFORM_VERSION }}"
+  become_user: "{{ test_build_server_user }}"
+
+- name: get xargs limit
+  shell: "xargs --show-limits"
+
+- name: Copy test-development-environment.sh to somewhere the jenkins user can access it
+  copy:
+    src: test-development-environment.sh
+    dest: "{{ test_build_server_repo_path }}"
+    mode: 0755
+
+- name: Validate build environment
+  shell: "bash test-development-environment.sh {{ item }}"
+  args:
+    chdir: "{{ test_build_server_repo_path }}/"
+  environment:
+    PYTHON_VERSION: "3.8"
+  become_user: "{{ test_build_server_user }}"
+  with_items:
+    - "unit"
+    - "js"

playbooks/vagrant-analytics.yml

@@ -7,6 +7,7 @@
     devstack: true
     disable_edx_services: true
     mongo_enable_journal: false
+    EDXAPP_NO_PREREQ_INSTALL: 0
     COMMON_SSH_PASSWORD_AUTH: "yes"
     EDXAPP_LMS_BASE: 127.0.0.1:8000
     EDXAPP_OAUTH_ENFORCE_SECURE: false

util/jenkins/app-container-provisioner.sh

@@ -84,10 +84,10 @@ chown :www-data /edx/etc/${app_service_name}.yml
 
 if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then
     # run migrations
-    docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database default
-    docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database default --noinput
-    docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database student_module_history
-    docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database student_module_history --noinput
+    docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database default
+    docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database default --noinput
+    docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database student_module_history
+    docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database student_module_history --noinput
 else
     # Run app migrations
     docker run --network=host --rm -u='www-data' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.settings.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_name}:/edx/var/${app_name} -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py migrate
@@ -123,7 +123,7 @@ fi
 if [[ ${app_service_name} == 'lms' ]]; then
     # temporary hack, create npm-install.log file
     touch /edx/app/edxapp/edx-platform/test_root/log/npm-install.log
-    docker run --network=host --rm -u='root' -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/${app_service_name}.yml -e CMS_CFG=/edx/etc/cms.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -e STATIC_ROOT_LMS=/edx/var/edxapp/staticfiles -e STATIC_ROOT_CMS=/edx/var/edxapp/staticfiles/studio -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /var/tmp/cms.yml:/edx/etc/cms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/${app_name}:/edx/var/${app_name} -v /edx/app/edxapp/edx-platform/test_root/log/npm-install.log:/edx/app/edxapp/edx-platform/test_root/log/npm-install.log -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest npm run build && ./manage.py lms collectstatic --noinput --settings=docker-production && ./manage.py cms collectstatic --noinput --settings=docker-production
+    docker run --network=host --rm -u='root' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/${app_service_name}.yml -e CMS_CFG=/edx/etc/cms.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /var/tmp/cms.yml:/edx/etc/cms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/${app_name}:/edx/var/${app_name} -v /edx/app/edxapp/edx-platform/test_root/log/npm-install.log:/edx/app/edxapp/edx-platform/test_root/log/npm-install.log -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest paver update_assets --debug-collect --settings=docker-production
 fi
 
 # Generate docker-compose file for app service