diff --git a/docker-compose-host.yml b/docker-compose-host.yml index ea558bbb97..5941bfbdc2 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -29,6 +29,9 @@ services: - edxapp_tox:/edx/app/edxapp/edx-platform/.tox - edxapp_uploads:/edx/var/edxapp/uploads - ${DEVSTACK_WORKSPACE}/src:/edx/src + lms-worker: + volumes: + - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform edx_notes_api: volumes: - ${DEVSTACK_WORKSPACE}/edx-notes-api:/edx/app/notes/ @@ -47,6 +50,9 @@ services: - edxapp_tox:/edx/app/edxapp/edx-platform/.tox - edxapp_uploads:/edx/var/edxapp/uploads - ${DEVSTACK_WORKSPACE}/src:/edx/src + cms-worker: + volumes: + - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform insights: volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights diff --git a/docker-compose.yml b/docker-compose.yml index 40bbbd2c66..4cb269ec59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -406,7 +406,8 @@ services: - "44567:4567" lms: - command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' + # Switch to `--settings devstack_with_worker` if you want to use lms-worker + command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" hostname: lms.devstack.edx depends_on: @@ -446,6 +447,22 @@ services: volumes: - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ + lms-worker: + command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && celery --app=lms.celery:APP worker -l debug -c 2' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms-worker" + hostname: lms-worker.devstack.edx + depends_on: + - mysql80 + - mysql57 + - redis + stdin_open: true + tty: true + image: edxops/lms-dev:latest + environment: + DJANGO_SETTINGS_MODULE: lms.envs.devstack_with_worker + # Dangerous to run Celery as root normally, but it's how we do things in devstack for some reason + C_FORCE_ROOT: "true" + insights: command: bash -c 'source /edx/app/insights/insights_env && while true; do python /edx/app/insights/insights/manage.py runserver 0.0.0.0:18110 --settings analytics_dashboard.settings.devstack; sleep 2; done' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights" @@ -584,7 +601,8 @@ services: - /edx/var/registrar/ cms: - command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' + # Switch to `--settings devstack_with_worker` if you want to use cms-worker + command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack; sleep 2; done' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.cms" hostname: cms.devstack.edx depends_on: @@ -623,6 +641,22 @@ services: volumes: - edxapp_cms_assets:/edx/var/edxapp/staticfiles/ + cms-worker: + command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && celery --app=cms.celery:APP worker -l debug -c 2' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms-worker" + hostname: cms-worker.devstack.edx + depends_on: + - mysql80 + - mysql57 + - redis + stdin_open: true + tty: true + image: edxops/lms-dev:latest + environment: + DJANGO_SETTINGS_MODULE: cms.envs.devstack_with_worker + # Dangerous to run Celery as root normally, but it's how we do things in devstack for some reason + C_FORCE_ROOT: "true" + xqueue: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue-dev:latest diff --git a/docs/advanced_configuration.rst b/docs/advanced_configuration.rst index 35b2b83ada..e3c3ed475a 100644 --- a/docs/advanced_configuration.rst +++ b/docs/advanced_configuration.rst @@ -25,3 +25,13 @@ This is handled for you automatically by setting the ``OPENEDX_RELEASE`` environ As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``. +Celery workers +~~~~~~~~~~~~~~ + +In devstack, edxapp and possibly other services are configured to run Celery tasks synchronously rather than requiring another worker service to handle them. However, there is some support for using a separate worker. To use Celery for LMS or CMS: + +1. Edit ``docker-compose.yml`` to update the LMS and CMS ``command`` lines to use ``--settings devstack_with_worker`` instead of just ``--settings devstack`` +2. Run ``make lms-worker-up cms-worker-up`` +3. Start LMS and CMS as usual + +To verify that Celery communication is working, try ``curl -sS 'http://localhost:18000/heartbeat?extended'`` and confirm that the ``celery`` component of the response shows success. diff --git a/docs/service_list.rst b/docs/service_list.rst index 0f9686ca9f..092b8028c6 100644 --- a/docs/service_list.rst +++ b/docs/service_list.rst @@ -55,13 +55,13 @@ Instead of a service name or list, you can also run commands like ``make dev.pro +------------------------------------+-------------------------------------+----------------+--------------+ | `xqueue`_ | http://localhost:18040/api/v1/ | Python/Django | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ -| `coursegraph` | http://localhost:7474/browser | Tooling (Java) | Extra | +| `coursegraph`_ | http://localhost:7474/browser | Tooling (Java) | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ -| `insights` | http://localhost:18110 | Python/Django | Extra | +| `insights`_ | http://localhost:18110 | Python/Django | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ -| `analyticsapi` | http://localhost:19001 | Python/Django | Extra | +| `analyticsapi`_ | http://localhost:19001 | Python/Django | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ -| `frontend-app-ora-grading` | http://localhost:1993 | MFE (React.js) | Extra | +| `frontend-app-ora-grading`_ | http://localhost:1993 | MFE (React.js) | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ Some common service combinations include: diff --git a/options.mk b/options.mk index 304cbbd59a..5a4d9b4e2c 100644 --- a/options.mk +++ b/options.mk @@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-authn+frontend- # Separated by plus signs. # Separated by plus signs. Listed in alphabetical order for clarity. EDX_SERVICES ?= \ -analyticsapi+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+cms+cms_watcher+xqueue+xqueue_consumer +analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer # Services with database migrations. # Should be a subset of $(EDX_SERVICES). @@ -76,7 +76,7 @@ analyticsapi+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-ac # Note: This list should contain _all_ db-backed services, even if not # configured to run; the list will be filtered later against $(DEFAULT_SERVICES). DB_SERVICES ?= \ -credentials+discovery+ecommerce+lms+registrar+cms +credentials+cms+discovery+ecommerce+lms+registrar # Services with static assets to be built. # Should be a subset of $(EDX_SERVICES). @@ -85,7 +85,7 @@ credentials+discovery+ecommerce+lms+registrar+cms # Note: This list should contain _all_ services with static asse to compile ts, even if not # configured to run; the list will be filtered later against $(DEFAULT_SERVICES). ASSET_SERVICES ?= \ -credentials+discovery+ecommerce+insights+lms+registrar+cms +credentials+cms+discovery+ecommerce+insights+lms+registrar # All third-party services. # Separated by plus signs. Listed in alphabetical order for clarity.