Skip to content

Commit

Permalink
chore: add enterprise catalog in devstack
Browse files Browse the repository at this point in the history
  • Loading branch information
huniafatima-arbi committed Nov 22, 2024
1 parent 3c79ad3 commit 343b4cd
Show file tree
Hide file tree
Showing 27 changed files with 2,243 additions and 3 deletions.
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,9 @@ dev.shell.analyticsapi:
dev.shell.insights:
docker compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash'

dev.shell.enterprise-catalog:
docker exec -it edx.devstack.enterprise-catalog env TERM=$(TERM) bash -c '/bin/bash'

dev.shell.%: ## Run a shell on the specified service's container.
docker compose exec $* /bin/bash

Expand Down
20 changes: 19 additions & 1 deletion docker-compose-host.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ services:
- discovery_node_modules:/edx/app/discovery/discovery/node_modules
- discovery_tox:/edx/app/discovery/discovery/.tox
- ${DEVSTACK_WORKSPACE}/src:/edx/src
- ${PWD}/py_configuration_files/course_discovery.py:/edx/app/discovery/discovery/course_discovery/settings/devstack.py
ecommerce:
volumes:
- ${DEVSTACK_WORKSPACE}/ecommerce:/edx/app/ecommerce/ecommerce
- ecommerce_node_modules:/edx/app/ecommerce/ecommerce/node_modules
- ecommerce_tox:/edx/app/ecommerce/ecommerce/.tox
- ${DEVSTACK_WORKSPACE}/src:/edx/src
- ${PWD}/py_configuration_files/ecommerce.py:/edx/app/ecommerce/ecommerce/ecommerce/settings/devstack.py
forum:
volumes:
- ${DEVSTACK_WORKSPACE}/cs_comments_service:/edx/app/forum/cs_comments_service
Expand All @@ -29,16 +31,20 @@ services:
- edxapp_tox:/edx/app/edxapp/edx-platform/.tox
- edxapp_uploads:/edx/var/edxapp/uploads
- ${DEVSTACK_WORKSPACE}/src:/edx/src
- ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py
- ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py
lms-worker:
volumes:
- ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
edx_notes_api:
volumes:
- ${DEVSTACK_WORKSPACE}/edx-notes-api:/edx/app/notes/
- ${DEVSTACK_WORKSPACE}/src:/edx/src
- ${PWD}/py_configuration_files/edx_notes_api.py:/edx/app/notes/notesserver/settings/devstack.py
registrar:
volumes:
- ${DEVSTACK_WORKSPACE}/registrar:/edx/app/registrar
- ${PWD}/py_configuration_files/registrar.py:/edx/app/registrar/registrar/registrar/settings/devstack.py
registrar-worker:
volumes:
- ${DEVSTACK_WORKSPACE}/registrar:/edx/app/registrar
Expand All @@ -50,18 +56,30 @@ services:
- edxapp_tox:/edx/app/edxapp/edx-platform/.tox
- edxapp_uploads:/edx/var/edxapp/uploads
- ${DEVSTACK_WORKSPACE}/src:/edx/src
- ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py
- ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py
cms-worker:
volumes:
- ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
insights:
volumes:
- ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights
- insights_node_modules:/edx/app/insights/insights/node_modules
- ${PWD}/py_configuration_files/analytics_dashboard.py:/edx/app/insights/edx_analytics_dashboard/edx_analytics_dashboard/settings/devstack.py
analyticsapi:
volumes:
- ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api
- ${DEVSTACK_WORKSPACE}/src:/edx/src

- ${PWD}/py_configuration_files/analytics_data_api.py:/edx/app/analytics_api/analytics_api/analyticsdataserver/settings/devstack.py
enterprise-catalog:
volumes:
- ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog
enterprise-catalog-worker:
volumes:
- ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog
enterprise-catalog-curations_worker:
volumes:
- ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog
# Note that frontends mount `src` to /edx/app/src instead of /edx/src.
# See ADR #5 for rationale.
frontend-app-account:
Expand Down
77 changes: 77 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -693,6 +693,82 @@ services:
aliases:
- edx.devstack.xqueue_consumer

enterprise-catalog:
image: edxops/enterprise-catalog-dev
container_name: edx.devstack.enterprise-catalog
hostname: enterprise-catalog.devstack.edx
command: bash -c 'while true; do python /edx/app/enterprise_catalog/enterprise_catalog/manage.py runserver 0.0.0.0:18160; sleep 2; done'
ports:
- "18160:18160"
depends_on:
- memcached
- mysql80
- enterprise-catalog-worker
networks:
default:
aliases:
- edx.devstack.enterprise-catalog
# Allows attachment to this container using 'docker attach <containerID>'.
stdin_open: true
tty: true
environment:
CELERY_ALWAYS_EAGER: 'false'
CELERY_BROKER_TRANSPORT: redis
CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379
CELERY_BROKER_VHOST: 0
CELERY_BROKER_PASSWORD: password
DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack
ENABLE_DJANGO_TOOLBAR: 1
DB_HOST: edx.devstack.mysql80
DB_NAME: enterprise_catalog
DB_PORT: 3306
DB_USER: catalog001
DB_PASSWORD: 'password'

enterprise-catalog-worker:
image: edxops/enterprise-catalog-dev
command: bash -c 'cd /edx/app/enterprise_catalog/enterprise_catalog && celery -A enterprise_catalog worker -Q enterprise_catalog.default -l DEBUG'
container_name: edx.devstack.enterprise.catalog.worker
depends_on:
- mysql80
environment:
CELERY_ALWAYS_EAGER: 'false'
CELERY_BROKER_TRANSPORT: redis
CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379
CELERY_BROKER_VHOST: 0
CELERY_BROKER_PASSWORD: password
DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack
COLUMNS: 80
hostname: worker.catalog.enterprise
ports:
- "18161:18161"
restart: always
# Allows attachment to this container using 'docker attach <containerID>'.
stdin_open: true
tty: true

enterprise-catalog-curations_worker:
image: edxops/enterprise-catalog-dev
command: bash -c 'cd /edx/app/enterprise_catalog/enterprise_catalog && celery -A enterprise_catalog worker -Q enterprise_catalog.curations -l DEBUG'
container_name: enterprise.catalog.curations
depends_on:
- mysql80
environment:
CELERY_ALWAYS_EAGER: 'false'
CELERY_BROKER_TRANSPORT: redis
CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379
CELERY_BROKER_VHOST: 0
CELERY_BROKER_PASSWORD: password
DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack
COLUMNS: 80
hostname: curations.catalog.enterprise
ports:
- "18162:18162"
restart: always
# Allows attachment to this container using 'docker attach <containerID>'.
stdin_open: true
tty: true

# ==========================================================================
# edX Microfrontends
#
Expand Down Expand Up @@ -910,3 +986,4 @@ volumes:
mysql57_data:
mysql80_data:
redis_data:
enterprise_catalog_mysql8:
3 changes: 3 additions & 0 deletions docs/service_list.rst
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro
+------------------------------------+-------------------------------------+----------------+--------------+
| `frontend-app-ora-grading`_ | http://localhost:1993 | MFE (React.js) | Extra |
+------------------------------------+-------------------------------------+----------------+--------------+
| `enterprise-catalog`_ | http://localhost:18160/ | Python/Django | Extra |
+------------------------------------+-------------------------------------+----------------+--------------+

Some common service combinations include:

Expand Down Expand Up @@ -95,3 +97,4 @@ Some common service combinations include:
.. _frontend-app-ora-grading: https://github.com/edx/frontend-app-ora-grading
.. _insights: https://github.com/edx/edx-analytics-dashboard
.. _analyticsapi: https://github.com/edx/edx-analytics-data-api
.. _enterprise-catalog: https://github.com/openedx/enterprise-catalog
2 changes: 1 addition & 1 deletion options.mk
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-authn+frontend-
# Separated by plus signs.
# Separated by plus signs. Listed in alphabetical order for clarity.
EDX_SERVICES ?= \
analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer
analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer+enterprise-catalog

# Services with database migrations.
# Should be a subset of $(EDX_SERVICES).
Expand Down
23 changes: 23 additions & 0 deletions provision-enterprise_catalog.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name="enterprise-catalog"
port="18160"

docker compose up -d $name

# Run migrations
echo -e "${GREEN}Running migrations for ${name}...${NC}"
docker exec -t enterprise.catalog.app bash -c "cd /edx/app/${name}/${name}/ && make migrate"

# Create superuser
echo -e "${GREEN}Creating super-user for ${name}...${NC}"
docker exec -t enterprise.catalog.app bash -c "echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"[email protected]\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None' | python /edx/app/${name}/${name}/manage.py shell"

# Provision IDA User in LMS
echo -e "${GREEN}Provisioning ${name}_worker in LMS...${NC}"
docker exec -t edx.devstack.lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user ${name}_worker ${name}[email protected] --staff --superuser"

# Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication.
docker exec -t edx.devstack.lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris 'http://localhost:${port}/complete/edx-oauth2/' --client-id '${name}-sso-key' --client-secret '${name}-sso-secret' --scopes 'user_id' ${name}-sso ${name}_worker"
docker exec -t edx.devstack.lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id '${name}-backend-service-key' --client-secret '${name}-backend-service-secret' ${name}-backend-service ${name}_worker"

# Restart enterprise.catalog app and worker containers
docker-compose restart app worker
4 changes: 4 additions & 0 deletions provision-mysql80.sql
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ CREATE DATABASE IF NOT EXISTS credentials;
CREATE USER IF NOT EXISTS 'credentials001'@'%' IDENTIFIED BY 'password';
GRANT ALL ON credentials.* TO 'credentials001'@'%';

CREATE DATABASE IF NOT EXISTS enterprise_catalog;
CREATE USER IF NOT EXISTS 'catalog001'@'%' IDENTIFIED BY 'password';
GRANT ALL ON enterprise_catalog.* TO 'catalog001'@'%';

CREATE DATABASE IF NOT EXISTS discovery;
CREATE USER IF NOT EXISTS 'discov001'@'%' IDENTIFIED BY 'password';
GRANT ALL ON discovery.* TO 'discov001'@'%';
Expand Down
3 changes: 2 additions & 1 deletion provision.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# This script will provision the services specified in the argument list,
# or all services if no arguments are provided.
#
# Non-existant services will be ignored.
# Non-existent services will be ignored.
# Specifying services more than once will cause them to be provisioned more
# than once.
#
Expand Down Expand Up @@ -49,6 +49,7 @@ xqueue \
coursegraph \
insights \
analyticsapi \
enterprise-catalog \
"

# What should we provision?
Expand Down
2 changes: 2 additions & 0 deletions provision.sql
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,7 @@ GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password';
CREATE DATABASE IF NOT EXISTS `reports_v1`;
GRANT ALL ON `reports_v1`.* TO 'analytics001'@'%' IDENTIFIED BY 'password';

CREATE DATABASE IF NOT EXISTS `enterprise_catalog`;
GRANT ALL ON `enterprise_catalog`.* TO 'catalog001'@'%' IDENTIFIED BY 'password';

FLUSH PRIVILEGES;
55 changes: 55 additions & 0 deletions py_configuration_files/analytics_data_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""Devstack settings."""

import os

from analyticsdataserver.settings.local import *

########## DATABASE CONFIGURATION
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'analytics-api',
'USER': 'api001',
'PASSWORD': 'password',
'HOST': 'edx.devstack.mysql',
'PORT': '3306',
},
'analytics_v1': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'reports_v1',
'USER': 'api001',
'PASSWORD': 'password',
'HOST': 'edx.devstack.mysql',
'PORT': '3306',
},
'analytics': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'reports',
'USER': 'reports001',
'PASSWORD': 'password',
'HOST': 'edx.devstack.mysql',
'PORT': '3306',
}
}

ANALYTICS_DATABASE_V1 = 'analytics_v1'

DB_OVERRIDES = dict(
USER=os.environ.get('DB_USER', DATABASES['default']['USER']),
PASSWORD=os.environ.get('DB_PASSWORD', DATABASES['default']['PASSWORD']),
HOST=os.environ.get('DB_HOST', DATABASES['default']['HOST']),
PORT=os.environ.get('DB_PORT', DATABASES['default']['PORT']),
)

for override, value in DB_OVERRIDES.items():
DATABASES['default'][override] = value
DATABASES['analytics'][override] = value
DATABASES['analytics_v1'][override] = value

DATABASE_ROUTERS = ['analyticsdataserver.router.AnalyticsAPIRouter', 'analyticsdataserver.router.AnalyticsModelsRouter']

########## END DATABASE CONFIGURATION

ALLOWED_HOSTS += ['edx.devstack.analyticsapi']

LMS_BASE_URL = "http://edx.devstack.lms:18000/"
47 changes: 47 additions & 0 deletions py_configuration_files/anayltics_dashboard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
"""django settings for development on devstack"""

from analytics_dashboard.settings.dev import *

DB_OVERRIDES = {
"PASSWORD": os.environ.get('DB_PASSWORD', DATABASES['default']['PASSWORD']),
"USER": os.environ.get('DB_USER', DATABASES['default']['USER']),
"NAME": os.environ.get('DB_NAME', DATABASES['default']['NAME']),
"HOST": os.environ.get('DB_HOST', DATABASES['default']['HOST']),
"PORT": os.environ.get('DB_PORT', DATABASES['default']['PORT']),
}

for override, value in DB_OVERRIDES.items():
DATABASES['default'][override] = value

DATA_API_URL = os.environ.get("API_SERVER_URL", 'http://edx.devstack.analyticsapi:19001/api/v0')
DATA_API_V1_ENABLED = True
DATA_API_URL_V1 = os.environ.get("API_SERVER_URL", 'http://edx.devstack.analyticsapi:19001/api/v1')

ENROLLMENT_AGE_AVAILABLE = True

# Set these to the correct values for your OAuth2/OpenID Connect provider (e.g., devstack)
SOCIAL_AUTH_EDX_OAUTH2_KEY = os.environ.get('SOCIAL_AUTH_EDX_OAUTH2_KEY', 'insights-sso-key')
SOCIAL_AUTH_EDX_OAUTH2_SECRET = os.environ.get('SOCIAL_AUTH_EDX_OAUTH2_SECRET', 'insights-sso-secret')
SOCIAL_AUTH_EDX_OAUTH2_ISSUER = os.environ.get('SOCIAL_AUTH_EDX_OAUTH2_ISSUER', 'http://localhost:18000')
SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT = os.environ.get('SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT', 'http://edx.devstack.lms:18000')
SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL = os.environ.get('SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL', 'http://localhost:18000/logout')
SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT = os.environ.get(
'SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT', 'http://localhost:18000',
)

BACKEND_SERVICE_EDX_OAUTH2_KEY = os.environ.get('BACKEND_SERVICE_EDX_OAUTH2_KEY', 'insights-backend-service-key')
BACKEND_SERVICE_EDX_OAUTH2_SECRET = os.environ.get(
'BACKEND_SERVICE_EDX_OAUTH2_SECRET', 'insights-backend-service-secret'
)
BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL = os.environ.get(
'BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL', 'http://edx.devstack.lms:18000/oauth2',
)

COURSE_API_URL = 'http://edx.devstack.lms:18000/api/courses/v1/'
GRADING_POLICY_API_URL = 'http://edx.devstack.lms:18000/api/grades/v1/'

MODULE_PREVIEW_URL = 'http://edx.devstack.lms:18000/xblock'

JWT_AUTH = {
'JWT_AUTH_HEADER_PREFIX': 'JWT',
}
Loading

0 comments on commit 343b4cd

Please sign in to comment.