diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 780d98f2a90a..808294b03052 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -56,9 +56,13 @@ jobs: - name: Install required Python dependencies run: | + # dev-requirements is needed because the linter will otherwise + # trip over some dev-only things like django-debug-toolbar + # (import debug_toolbar) that aren't in testing.txt. make dev-requirements pip uninstall -y mysqlclient pip install --no-binary mysqlclient mysqlclient + pip check - name: Run quality tests run: | diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index e28be7aba70e..f6e99ffc92ea 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -57,19 +57,16 @@ jobs: - name: Install Required Python Dependencies env: - PIP_SRC_DIR: ${{ runner.temp }} + PIP_SRC: ${{ runner.temp }} run: | - pip install -r requirements/pip.txt - pip install -r requirements/edx/testing.txt --src $PIP_SRC_DIR - pip install -e . --src $PIP_SRC_DIR - pip install -r requirements/edx/django.txt + make test-requirements - name: Run Quality Tests env: TEST_SUITE: quality SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh SHARD: 4 - PIP_SRC_DIR: ${{ runner.temp }} + PIP_SRC: ${{ runner.temp }} TARGET_BRANCH: ${{ github.base_ref }} run: | ./scripts/all-tests.sh diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index 94473111404d..fc5f9bee6371 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -15,7 +15,8 @@ jobs: fail-fast: false matrix: python-version: [ '3.8' ] - django-version: [ "3.2" ] + django-version: + - "pinned" shard_name: [ "lms-1", "lms-2", @@ -60,16 +61,18 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} restore-keys: ${{ runner.os }}-pip- - name: Install Required Python Dependencies + env: + PIP_SRC: ${{ runner.temp }} run: | - pip install -r requirements/pip.txt - pip install -r requirements/edx/development.txt --src ${{ runner.temp }} - # edx-platform installs some Python projects from within the edx-platform repo itself. - pip install -e . + make test-requirements + if [[ "${{ matrix.django-version }}" != "pinned" ]]; then pip install "django~=${{ matrix.django-version }}.0" + pip check # fail if this test-reqs/Django combination is broken + fi - name: Setup and run tests uses: ./.github/actions/unit-tests @@ -80,7 +83,8 @@ jobs: strategy: matrix: python-version: [ '3.8' ] - django-version: [ "3.2" ] + django-version: + - "pinned" steps: - uses: actions/checkout@v2 @@ -102,14 +106,16 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} restore-keys: ${{ runner.os }}-pip- - name: Install Required Python Dependencies run: | - pip install -r requirements/pip.txt - make dev-requirements + make test-requirements + if [[ "${{ matrix.django-version }}" != "pinned" ]]; then pip install "django~=${{ matrix.django-version }}.0" + pip check # fail if this test-reqs/Django combination is broken + fi - name: verify unit tests count uses: ./.github/actions/verify-tests-count diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 66cb5f7fd13f..54ad4f5891dc 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -62,10 +62,9 @@ jobs: - name: install requirements run: | sudo make test-requirements - if [[ "${{ matrix.django-version }}" == "pinned" ]]; then - sudo pip install -r requirements/edx/django.txt - else + if [[ "${{ matrix.django-version }}" != "pinned" ]]; then sudo pip install "django~=${{ matrix.django-version }}.0" + sudo pip check # fail if this test-reqs/Django combination is broken fi - name: list installed package versions diff --git a/Makefile b/Makefile index dd97cbb6b960..7aa7334fdfac 100644 --- a/Makefile +++ b/Makefile @@ -118,12 +118,6 @@ $(COMMON_CONSTRAINTS_TXT): compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt - @# This is a temporary solution to override the real common_constraints.txt - @# In edx-lint, until the pyjwt constraint in edx-lint has been removed. - @# See BOM-2721 for more details. - sed 's/Django<2.3//g' requirements/common_constraints.txt > requirements/common_constraints.tmp - mv requirements/common_constraints.tmp requirements/common_constraints.txt - pip install -q pip-tools pip-compile --allow-unsafe --upgrade -o requirements/edx/pip.txt requirements/edx/pip.in @@ -139,11 +133,6 @@ compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements pip install -qr requirements/edx/pip.txt pip install -qr requirements/edx/pip-tools.txt - # Let tox control the Django version for tests - grep -e "^django==" requirements/edx/base.txt > requirements/edx/django.txt - sed '/^[dD]jango==/d' requirements/edx/testing.txt > requirements/edx/testing.tmp - mv requirements/edx/testing.tmp requirements/edx/testing.txt - upgrade: pre-requirements ## update the pip requirements files to use the latest releases satisfying our constraints $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py index 3b588f65faf6..33cbab7ef987 100644 --- a/pavelib/prereqs.py +++ b/pavelib/prereqs.py @@ -24,10 +24,8 @@ # a corresponding change to circle.yml, which is how the python # prerequisites are installed for builds on circleci.com toxenv = os.environ.get('TOXENV') -if toxenv and toxenv != 'quality-django32': +if toxenv and toxenv != 'quality': PYTHON_REQ_FILES = ['requirements/edx/testing.txt'] -elif toxenv and toxenv == 'quality-django32': - PYTHON_REQ_FILES = ['requirements/edx/testing.txt', 'requirements/edx/django.txt'] else: PYTHON_REQ_FILES = ['requirements/edx/development.txt'] @@ -173,11 +171,7 @@ def python_prereqs_installation(): def pip_install_req_file(req_file): """Pip install the requirements file.""" pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w' - - if Env.PIP_SRC_DIR: - sh(f"{pip_cmd} -r {req_file} --src {Env.PIP_SRC_DIR}") - else: - sh(f"{pip_cmd} -r {req_file}") + sh(f"{pip_cmd} -r {req_file}") @task @@ -313,8 +307,8 @@ def install_python_prereqs(): files_to_fingerprint.append(sysconfig.get_python_lib()) # In a virtualenv, "-e installs" get put in a src directory. - if Env.PIP_SRC_DIR: - src_dir = Env.PIP_SRC_DIR + if Env.PIP_SRC: + src_dir = Env.PIP_SRC else: src_dir = os.path.join(sys.prefix, "src") if os.path.isdir(src_dir): diff --git a/pavelib/tests.py b/pavelib/tests.py index 28988e8f05e9..39c0fb4f7abd 100644 --- a/pavelib/tests.py +++ b/pavelib/tests.py @@ -32,10 +32,6 @@ ("test-id=", "t", "Test id"), ("fail-fast", "x", "Fail suite on first failed test"), ("fasttest", "a", "Run without collectstatic"), - make_option( - "--django_version", dest="django_version", - help="Run against which Django version (3.2)." - ), make_option( "--eval-attr", dest="eval_attr", help="Only run tests matching given attribute expression." @@ -106,10 +102,8 @@ def test_system(options, passthrough_options): """ system = getattr(options, 'system', None) test_id = getattr(options, 'test_id', None) - django_version = getattr(options, 'django_version', None) assert system in (None, 'lms', 'cms') - assert django_version in (None, '3.2') if hasattr(options.test_system, 'with_wtw'): call_task('fetch_coverage_test_selection_data', options={ @@ -161,10 +155,6 @@ def test_system(options, passthrough_options): ("test-id=", "t", "Test id"), ("failed", "f", "Run only failed tests"), ("fail-fast", "x", "Run only failed tests"), - make_option( - "--django_version", dest="django_version", - help="Run against which Django version (3.2)." - ), make_option( "--eval-attr", dest="eval_attr", help="Only run tests matching given attribute expression." @@ -201,9 +191,6 @@ def test_lib(options, passthrough_options): """ lib = getattr(options, 'lib', None) test_id = getattr(options, 'test_id', lib) - django_version = getattr(options, 'django_version', None) - - assert django_version in (None, '3.2') if test_id: # Testing a single test id. diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index ef8ce87e5662..9e2e2fe5b831 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -216,8 +216,9 @@ class Env: # Directory for i18n test reports I18N_REPORT_DIR = REPORT_DIR / 'i18n' - # Directory for keeping src folder that comes with pip installation - PIP_SRC_DIR = os.environ.get("PIP_SRC_DIR") + # Directory for keeping src folder that comes with pip installation. + # Setting this is equivalent to passing `--src