Skip to content

Commit

Permalink
Merge branch 'master' into JF/fix_certificate_palm
Browse files Browse the repository at this point in the history
  • Loading branch information
BetoFandino authored Sep 26, 2023
2 parents dae1c75 + 437b7e6 commit b9e1819
Show file tree
Hide file tree
Showing 312 changed files with 5,745 additions and 4,399 deletions.
30 changes: 22 additions & 8 deletions .github/workflows/check-consistent-dependencies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,6 @@ name: Consistent Python dependencies

on:
pull_request:
paths:
- 'requirements/**'
push:
branches:
- master
paths:
- 'requirements/**'

defaults:
run:
Expand All @@ -25,16 +18,37 @@ jobs:
runs-on: ubuntu-22.04

steps:
# Only run remaining steps if there are changes to requirements/**
- name: "Decide whether to short-circuit"
env:
GH_TOKEN: "${{ github.token }}"
PR_URL: "${{ github.event.pull_request.html_url }}"
run: |
paths=$(gh pr diff "$PR_URL" --name-only)
echo $'Paths touched in PR:\n'"$paths"
# The ^"? is because git may quote weird file paths
matched="$(echo "$paths" | grep -P '^"?requirements/' || true)"
echo $'Relevant paths:\n'"$matched"
if [[ -n "$matched" ]]; then
echo "RELEVANT=true" >> "$GITHUB_ENV"
fi
- uses: actions/checkout@v3
if: ${{ env.RELEVANT == 'true' }}

- uses: actions/setup-python@v4
if: ${{ env.RELEVANT == 'true' }}
with:
python-version: '3.8'

- run: |
- name: "Recompile requirements"
if: ${{ env.RELEVANT == 'true' }}
run: |
make compile-requirements
- name: Fail if compiling requirements caused changes
if: ${{ env.RELEVANT == 'true' }}
run: |
SUMMARY_HELP=$(cat <<'EOMARKDOWN'
# Inconsistent Python dependencies
Expand Down
72 changes: 72 additions & 0 deletions .github/workflows/compile-python-requirements.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: Recompile Python dependencies

on:
workflow_dispatch:
inputs:
branch:
description: 'Target branch to create requirements PR against'
required: true
default: 'master'
type: string

defaults:
run:
shell: bash # making this explicit opts into -e -o pipefail

jobs:
recompile-python-dependencies:
runs-on: ubuntu-20.04

steps:
- name: Check out target branch
uses: actions/checkout@v3
with:
ref: "${{ inputs.branch }}"

- name: Set up Python environment
uses: actions/setup-python@v4
with:
python-version: "3.8"

- name: Run make compile-requirements
env:
PACKAGE: "${{ inputs.package }}"
run: |
make compile-requirements
- name: PR preflight
run: |
if git diff --exit-code; then
# Fail early (and avoid quiet failure of create-pull-request action)
echo "Error: No changes, so not creating PR." | tee -a "$GITHUB_STEP_SUMMARY"
exit 1
fi
- name: Make a PR
id: make-pr
uses: peter-evans/create-pull-request@v5
with:
branch: "${{ github.triggering_actor }}/compile-python-deps"
branch-suffix: short-commit-hash
add-paths: requirements
commit-message: |
feat: Recompile Python dependencies
Commit generated by workflow `${{ github.workflow_ref }}`
title: "chore: Recompile Python dependencies"
body: >-
PR generated by workflow `${{ github.workflow_ref }}`
on behalf of @${{ github.triggering_actor }}.
assignees: "${{ github.triggering_actor }}"
reviewers: "${{ github.triggering_actor }}"

- name: Job summary
env:
PR_URL: "${{ steps.make-pr.outputs.pull-request-url }}"
run: |
if [[ -z "$PR_URL" ]]; then
echo "PR not created; see log for more information" | tee -a "$GITHUB_STEP_SUMMARY"
exit 1
else
echo "PR created or updated: $PR_URL" | tee -a "$GITHUB_STEP_SUMMARY"
fi
33 changes: 33 additions & 0 deletions .github/workflows/migrations-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,17 @@ jobs:
matrix:
os: [ ubuntu-20.04 ]
python-version: [ 3.8 ]
# 'pinned' is used to install the latest patch version of Django
# within the global constraint i.e. Django==3.2.21 in current case
# because we have global constraint of Django<4.2
django-version: ["pinned", "4.2"]
mongo-version: ["4"]
mysql-version: ["5.7", "8"]
# excluding mysql5.7 with Django 4.2 since Django 4.2 has
# dropped support for MySQL<8
exclude:
- django-version: "4.2"
mysql-version: "5.7"
services:
mongo:
image: mongo:${{ matrix.mongo-version }}
Expand Down Expand Up @@ -92,6 +101,14 @@ jobs:
- name: Install Python dependencies
run: |
make dev-requirements
if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
pip install "django~=${{ matrix.django-version }}.0"
pip check # fail if this test-reqs/Django combination is broken
fi
- name: list installed package versions
run: |
sudo pip freeze
- name: Run Tests
env:
Expand All @@ -103,3 +120,19 @@ jobs:
./manage.py lms migrate
echo "Running the CMS migrations."
./manage.py cms migrate
# This job aggregates test results. It's the required check for branch protection.
# https://github.com/marketplace/actions/alls-green#why
# https://github.com/orgs/community/discussions/33579
success:
name: Migrations checks successful
if: always()
needs:
- check_migrations
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
# uses: re-actors/[email protected]
uses: re-actors/alls-green@13b4244b312e8a314951e03958a2f91519a6a3c9
with:
jobs: ${{ toJSON(needs) }}
2 changes: 1 addition & 1 deletion .github/workflows/pylint-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
- module-name: openedx-1
path: "--django-settings-module=lms.envs.test openedx/core/types/ openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/content_staging/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/djangoapps/course_live/"
- module-name: openedx-2
path: "--django-settings-module=lms.envs.test openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/learner_pathway/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/"
path: "--django-settings-module=lms.envs.test openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/learner_pathway/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/"
- module-name: common
path: "--django-settings-module=lms.envs.test common"
- module-name: cms
Expand Down
45 changes: 45 additions & 0 deletions .github/workflows/semgrep.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Finds code problems by structural pattern matching.
#
# New rules can be added to test_root/semgrep/ and they should be picked up
# automatically. See https://semgrep.dev/docs/ for documentation.

name: Semgrep code quality

on:
pull_request:
push:
branches:
- master

jobs:
run_semgrep:
name: Semgrep analysis
runs-on: "${{ matrix.os }}"
strategy:
matrix:
os: [ "ubuntu-20.04" ]
python-version: [ "3.8" ]

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1

- uses: actions/setup-python@v4
with:
python-version: "${{ matrix.python-version }}"

- name: Install semgrep
run: |
make pre-requirements
pip-sync requirements/edx/semgrep.txt
- name: Run semgrep
env:
# Peg this to some reasonable value so that semgrep's rewrapping
# of messages doesn't break up lines in an unpredictable manner:
# https://github.com/returntocorp/semgrep/issues/8608
COLUMNS: 80
run: |
semgrep scan --config test_root/semgrep/ --error --quiet \
-- lms cms common openedx
1 change: 1 addition & 0 deletions .github/workflows/unit-test-shards.json
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@
"openedx-4": {
"settings": "cms.envs.test",
"paths": [
"openedx/core/djangoapps/content_tagging/",
"openedx/core/djangoapps/geoinfo/",
"openedx/core/djangoapps/header_control/",
"openedx/core/djangoapps/heartbeat/",
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
- "3.8"
django-version:
- "pinned"
#- "4.0"
- "4.2"
# When updating the shards, remember to make the same changes in
# .github/workflows/unit-tests-gh-hosted.yml
shard_name:
Expand Down
45 changes: 23 additions & 22 deletions .github/workflows/upgrade-one-python-dependency.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,32 @@ name: Upgrade one Python dependency

on:
workflow_dispatch:
inputs:
branch:
description: 'Target branch to create requirements PR against'
required: true
default: 'master'
type: string
package:
description: 'Name of package to upgrade'
required: true
type: string
version:
description: 'Version number to upgrade to in constraints.txt (only needed if pinned)'
default: ''
type: string
change_desc:
description: |
Description of change, for commit message and PR. (What does the new version add or fix?)
default: ''
type: string
inputs:
branch:
description: 'Target branch to create requirements PR against'
required: true
default: 'master'
type: string
package:
description: 'Name of package to upgrade'
required: true
type: string
version:
description: 'Version number to upgrade to in constraints.txt (only needed if pinned)'
default: ''
type: string
change_desc:
description: |
Description of change, for commit message and PR. (What does the new version add or fix?)
default: ''
type: string

defaults:
run:
shell: bash # making this explicit opts into -e -o pipefail

jobs:
upgrade-one-python-dependency-workflow:
upgrade-one-python-dependency:
runs-on: ubuntu-20.04

steps:
Expand Down Expand Up @@ -92,8 +92,9 @@ jobs:
${{ env.body_prefix }}Commit generated by workflow `${{ github.workflow_ref }}`
title: "feat: Upgrade Python dependency ${{ inputs.package }}"
body: |
${{ env.body_prefix }}PR generated by workflow `${{ github.workflow_ref }}` on behalf of @${{ github.triggering_actor }}.
body: >-
${{ env.body_prefix }}PR generated by workflow `${{ github.workflow_ref }}`
on behalf of @${{ github.triggering_actor }}.
assignees: "${{ github.triggering_actor }}"
reviewers: "${{ github.triggering_actor }}"

Expand Down
15 changes: 11 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,11 @@ COPY . .
# Install Python requirements again in order to capture local projects
RUN pip install -e .

USER app

# Production target
FROM base as production

USER app

ENV EDX_PLATFORM_SETTINGS='docker-production'
ENV SERVICE_VARIANT "${SERVICE_VARIANT}"
ENV SERVICE_PORT "${SERVICE_PORT}"
Expand All @@ -167,9 +168,15 @@ CMD gunicorn \
# Development target
FROM base as development

COPY --from=builder-development /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp
RUN apt-get update && \
apt-get -y install --no-install-recommends \
# wget is used in Makefile for common_constraints.txt
wget \
&& \
apt-get clean all && \
rm -rf /var/lib/apt/*

USER root
COPY --from=builder-development /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp

RUN ln -s "$(pwd)/lms/envs/devstack-experimental.yml" "$LMS_CFG"
RUN ln -s "$(pwd)/cms/envs/devstack-experimental.yml" "$CMS_CFG"
Expand Down
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ REQ_FILES = \
requirements/edx/testing \
requirements/edx/development \
requirements/edx/assets \
requirements/edx/semgrep \
scripts/xblock/requirements

define COMMON_CONSTRAINTS_TEMP_COMMENT
Expand All @@ -111,7 +112,7 @@ endef
COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
.PHONY: $(COMMON_CONSTRAINTS_TXT)
$(COMMON_CONSTRAINTS_TXT):
wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)"
wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
printf "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)

compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
Expand Down
10 changes: 5 additions & 5 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ platform. Functionally, the edx-platform repository provides two services:
* CMS (Content Management Service), which powers Open edX Studio, the platform's learning content authoring environment; and
* LMS (Learning Management Service), which delivers learning content.

Documentation
*************

Documentation can be found at https://docs.openedx.org/projects/edx-platform.

Getting Started
***************

Expand Down Expand Up @@ -81,11 +86,6 @@ and other rich community resources.

.. _Open edX site: https://openedx.org

Documentation
*************

Documentation can be found at https://docs.openedx.org.


Getting Help
************
Expand Down
Loading

0 comments on commit b9e1819

Please sign in to comment.