diff --git a/.dockerignore b/.dockerignore index 1665783ba48e..e8f6e66311c7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -89,6 +89,7 @@ test_root/paver_logs/ test_root/uploads/ **/django-pyfs **/.tox/ +common/test/db_cache/bok_choy_*.yaml common/test/data/badges/*.png ### Installation artifacts diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 271b6ada81ee..985dd0aab56e 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -64,8 +64,8 @@ jobs: - name: Run Static Assets Check env: - LMS_CFG: lms/envs/minimal.yml - CMS_CFG: lms/envs/minimal.yml + LMS_CFG: lms/envs/bok_choy.yml + CMS_CFG: cms/envs/bok_choy.yml run: | paver update_assets lms diff --git a/.gitignore b/.gitignore index 169296690009..1866979edc93 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ test_root/paver_logs/ test_root/uploads/ django-pyfs .tox/ +common/test/db_cache/bok_choy_*.yaml common/test/data/badges/*.png ### Installation artifacts diff --git a/cms/envs/bok_choy.auth.json b/cms/envs/bok_choy.auth.json new file mode 100644 index 000000000000..db92b5ba924e --- /dev/null +++ b/cms/envs/bok_choy.auth.json @@ -0,0 +1,116 @@ +{ + "AWS_ACCESS_KEY_ID": "", + "AWS_SECRET_ACCESS_KEY": "", + "CELERY_BROKER_PASSWORD": "celery", + "CELERY_BROKER_USER": "celery", + "CONTENTSTORE": { + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "localhost" + ], + "port": 27017 + }, + "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", + "OPTIONS": { + "db": "test", + "host": [ + "localhost" + ], + "port": 27017 + } + }, + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.mysql", + "HOST": "localhost", + "NAME": "edxtest", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + }, + "student_module_history": { + "ENGINE": "django.db.backends.mysql", + "HOST": "localhost", + "NAME": "student_module_history_test", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + } + }, + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "localhost" + ], + "port": 27017 + }, + "JWT_AUTH": { + "JWT_SECRET_KEY": "super-secret-key", + "JWT_PUBLIC_SIGNING_JWK_SET": "{\"keys\": [{\"kid\": \"BTZ9HA6K\", \"e\": \"AQAB\", \"kty\": \"RSA\", \"n\": \"o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ\"}]}" + }, + "MODULESTORE": { + "default": { + "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore", + "OPTIONS": { + "mappings": {}, + "stores": [ + { + "NAME": "draft", + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "localhost" + ], + "port": 27017 + }, + "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore", + "OPTIONS": { + "collection": "modulestore", + "db": "test", + "default_class": "xmodule.hidden_block.HiddenBlock", + "fs_root": "** OVERRIDDEN **", + "host": [ + "localhost" + ], + "port": 27017, + "render_template": "common.djangoapps.edxmako.shortcuts.render_to_string" + } + }, + { + "NAME": "xml", + "ENGINE": "xmodule.modulestore.xml.XMLModuleStore", + "OPTIONS": { + "data_dir": "** OVERRIDDEN **", + "default_class": "xmodule.hidden_block.HiddenBlock" + } + } + ] + } + } + }, + "DJFS": { + "type": "s3fs", + "bucket": "test", + "prefix": "test", + "aws_access_key_id": "test", + "aws_secret_access_key": "test" + }, + "SECRET_KEY": "", + "XQUEUE_INTERFACE": { + "basic_auth": [ + "edx", + "edx" + ], + "django_auth": { + "password": "password", + "username": "lms" + }, + "url": "http://localhost:18040" + }, + "ZENDESK_API_KEY": "", + "ZENDESK_USER": "" +} diff --git a/cms/envs/bok_choy.env.json b/cms/envs/bok_choy.env.json new file mode 100644 index 000000000000..4defe0ccc154 --- /dev/null +++ b/cms/envs/bok_choy.env.json @@ -0,0 +1,132 @@ +{ + "BUGS_EMAIL": "bugs@example.com", + "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com", + "CACHES": { + "celery": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "integration_celery", + "LOCATION": [ + "localhost:11211" + ] + }, + "default": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "sandbox_default", + "LOCATION": [ + "localhost:11211" + ] + }, + "general": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "sandbox_general", + "LOCATION": [ + "localhost:11211" + ] + }, + "mongo_metadata_inheritance": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "integration_mongo_metadata_inheritance", + "LOCATION": [ + "localhost:11211" + ] + }, + "staticfiles": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "integration_static_files", + "LOCATION": [ + "localhost:11211" + ] + } + }, + "CELERY_ALWAYS_EAGER": true, + "CELERY_BROKER_HOSTNAME": "localhost", + "CELERY_BROKER_TRANSPORT": "amqp", + "CERT_QUEUE": "certificates", + "CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION": false, + "CMS_BASE": "localhost:8031", + "CODE_JAIL": { + "limits": { + "REALTIME": 3, + "VMEM": 0 + } + }, + "COMMENTS_SERVICE_KEY": "password", + "COMMENTS_SERVICE_URL": "http://localhost:4567", + "CONTACT_EMAIL": "info@example.com", + "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com", + "DEFAULT_FROM_EMAIL": "registration@example.com", + "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", + "SOCIAL_SHARING_SETTINGS": { + "CUSTOM_COURSE_URLS": true + }, + "FEATURES": { + "CERTIFICATES_HTML_VIEW": true, + "ENABLE_DISCUSSION_SERVICE": true, + "ENABLE_GRADE_DOWNLOADS": true, + "ENTRANCE_EXAMS": true, + "MILESTONES_APP": true, + "PREVIEW_LMS_BASE": "preview.localhost:8003", + "ENABLE_CONTENT_LIBRARIES": true, + "ENABLE_SPECIAL_EXAMS": true, + "SHOW_HEADER_LANGUAGE_SELECTOR": true, + "ENABLE_EXTENDED_COURSE_DETAILS": true, + "CUSTOM_COURSES_EDX": true + }, + "FEEDBACK_SUBMISSION_EMAIL": "", + "GITHUB_REPO_ROOT": "** OVERRIDDEN **", + "GRADES_DOWNLOAD": { + "BUCKET": "edx-grades", + "ROOT_PATH": "/tmp/edx-s3/grades", + "STORAGE_TYPE": "localfs" + }, + "LMS_BASE": "localhost:8003", + "LMS_ROOT_URL": "http://localhost:8003", + "LOCAL_LOGLEVEL": "INFO", + "LOGGING_ENV": "sandbox", + "LOG_DIR": "** OVERRIDDEN **", + "MEDIA_URL": "/media/", + "MKTG_URL_LINK_MAP": {}, + "SERVER_EMAIL": "devops@example.com", + "SESSION_COOKIE_DOMAIN": null, + "SITE_NAME": "localhost", + "STATIC_URL_BASE": "/static/", + "SYSLOG_SERVER": "", + "TECH_SUPPORT_EMAIL": "technical@example.com", + "TIME_ZONE": "America/New_York", + "WIKI_ENABLED": true, +} diff --git a/cms/envs/bok_choy.py b/cms/envs/bok_choy.py new file mode 100644 index 000000000000..b6809dd48f0a --- /dev/null +++ b/cms/envs/bok_choy.py @@ -0,0 +1,181 @@ +""" +Settings for Bok Choy tests that are used when running Studio. + +Bok Choy uses two different settings files: +1. test_static_optimized is used when invoking collectstatic +2. bok_choy is used when running the tests + +Note: it isn't possible to have a single settings file, because Django doesn't +support both generating static assets to a directory and also serving static +from the same directory. +""" + + +# Silence noisy logs +import logging +import os + +from django.utils.translation import gettext_lazy +from path import Path as path + +from openedx.core.release import RELEASE_LINE +from xmodule.modulestore.modulestore_settings import update_module_store_settings # lint-amnesty, pylint: disable=wrong-import-order + +########################## Prod-like settings ################################### +# These should be as close as possible to the settings we use in production. +# As in prod, we read in environment and auth variables from JSON files. +# Unlike in prod, we use the JSON files stored in this repo. +# This is a convenience for ensuring (a) that we can consistently find the files +# and (b) that the files are the same in Jenkins as in local dev. +os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy' +CONFIG_ROOT = path(__file__).abspath().dirname() +os.environ['STUDIO_CFG'] = str.format("{config_root}/{service_variant}.yml", + config_root=CONFIG_ROOT, + service_variant=os.environ['SERVICE_VARIANT']) +os.environ['REVISION_CFG'] = f"{CONFIG_ROOT}/revisions.yml" + +from .production import * # pylint: disable=wildcard-import, unused-wildcard-import, wrong-import-position + + +######################### Testing overrides #################################### + +# Redirect to the test_root folder within the repo +TEST_ROOT = REPO_ROOT / "test_root" +GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath() +LOG_DIR = (TEST_ROOT / "log").abspath() +DATA_DIR = TEST_ROOT / "data" + +# Configure modulestore to use the test folder within the repo +update_module_store_settings( + MODULESTORE, + module_store_options={ + 'fs_root': (TEST_ROOT / "data").abspath(), + }, + xml_store_options={ + 'data_dir': (TEST_ROOT / "data").abspath(), + }, + default_store=os.environ.get('DEFAULT_STORE', 'draft'), +) + +# Needed to enable licensing on video blocks +XBLOCK_SETTINGS.update({'VideoBlock': {'licensing_enabled': True}}) + +# Capture the console log via template includes, until webdriver supports log capture again +CAPTURE_CONSOLE_LOG = True + +PLATFORM_NAME = gettext_lazy("édX") +PLATFORM_DESCRIPTION = gettext_lazy("Open édX Platform") +STUDIO_NAME = gettext_lazy("Your Platform 𝓢𝓽𝓾𝓭𝓲𝓸") +STUDIO_SHORT_NAME = gettext_lazy("𝓢𝓽𝓾𝓭𝓲𝓸") + +############################ STATIC FILES ############################# + +# Enable debug so that static assets are served by Django +DEBUG = True + +# Serve static files at /static directly from the staticfiles directory under test root +# Note: optimized files for testing are generated with settings from test_static_optimized +STATIC_URL = "/static/" +STATICFILES_FINDERS = [ + 'django.contrib.staticfiles.finders.FileSystemFinder', +] +STATICFILES_DIRS = [ + (TEST_ROOT / "staticfiles" / "cms").abspath(), +] + +DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' +MEDIA_ROOT = TEST_ROOT / "uploads" + +WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = TEST_ROOT / "staticfiles" / "cms" / "webpack-stats.json" + +LOG_OVERRIDES = [ + ('common.djangoapps.track.middleware', logging.CRITICAL), + ('edx.discussion', logging.CRITICAL), +] +for log_name, log_level in LOG_OVERRIDES: + logging.getLogger(log_name).setLevel(log_level) + +# Use the auto_auth workflow for creating users and logging them in +FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True +FEATURES['RESTRICT_AUTOMATIC_AUTH'] = False + +# Enable milestones app +FEATURES['MILESTONES_APP'] = True + +# Enable pre-requisite course +FEATURES['ENABLE_PREREQUISITE_COURSES'] = True + +# Enable student notes +FEATURES['ENABLE_EDXNOTES'] = True + +# Enable teams feature +FEATURES['ENABLE_TEAMS'] = True + +# Enable custom content licensing +FEATURES['LICENSING'] = True + +FEATURES['ENABLE_MOBILE_REST_API'] = True # Enable video bumper in Studio +FEATURES['ENABLE_VIDEO_BUMPER'] = True # Enable video bumper in Studio settings + +FEATURES['ENABLE_ENROLLMENT_TRACK_USER_PARTITION'] = True + +# Whether archived courses (courses with end dates in the past) should be +# shown in Studio in a separate list. +FEATURES['ENABLE_SEPARATE_ARCHIVED_COURSES'] = True + +# Enable partner support link in Studio footer +PARTNER_SUPPORT_EMAIL = 'partner-support@example.com' + +########################### Entrance Exams ################################# +FEATURES['ENTRANCE_EXAMS'] = True + +FEATURES['ENABLE_SPECIAL_EXAMS'] = True + +# Point the URL used to test YouTube availability to our stub YouTube server +YOUTUBE_PORT = 9080 +YOUTUBE['TEST_TIMEOUT'] = 5000 +YOUTUBE_HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') +YOUTUBE['API'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/get_youtube_api/" +YOUTUBE['METADATA_URL'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/test_youtube/" + +FEATURES['ENABLE_COURSEWARE_INDEX'] = True +FEATURES['ENABLE_LIBRARY_INDEX'] = True +FEATURES['ENABLE_CONTENT_LIBRARY_INDEX'] = False + +ORGANIZATIONS_AUTOCREATE = False + +SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine" +# Path at which to store the mock index +MOCK_SEARCH_BACKING_FILE = ( + TEST_ROOT / "index_file.dat" +).abspath() + +# this secret key should be the same as lms/envs/bok_choy.py's +SECRET_KEY = "very_secret_bok_choy_key" + +LMS_ROOT_URL = "http://localhost:8003" +if RELEASE_LINE == "master": + # On master, acceptance tests use edX books, not the default Open edX books. + HELP_TOKENS_BOOKS = { + 'learner': 'https://edx.readthedocs.io/projects/edx-guide-for-students', + 'course_author': 'https://edx.readthedocs.io/projects/edx-partner-course-staff', + } + +########################## VIDEO TRANSCRIPTS STORAGE ############################ +VIDEO_TRANSCRIPTS_SETTINGS = dict( + VIDEO_TRANSCRIPTS_MAX_BYTES=3 * 1024 * 1024, # 3 MB + STORAGE_KWARGS=dict( + location=MEDIA_ROOT, + base_url=MEDIA_URL, + ), + DIRECTORY_PREFIX='video-transcripts/', +) + +INSTALLED_APPS.append('openedx.testing.coverage_context_listener') + +##################################################################### +# Lastly, see if the developer has any local overrides. +try: + from .private import * # pylint: disable=wildcard-import +except ImportError: + pass diff --git a/cms/envs/bok_choy.yml b/cms/envs/bok_choy.yml new file mode 100644 index 000000000000..735a94c4c2c9 --- /dev/null +++ b/cms/envs/bok_choy.yml @@ -0,0 +1,155 @@ +# ingested bok_choy.env.json +# ingested bok_choy.auth.json +AWS_ACCESS_KEY_ID: '' +AWS_SECRET_ACCESS_KEY: '' +BUGS_EMAIL: bugs@example.com +BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com +CACHES: + celery: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + ignore_exc: true + no_delay: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_celery + LOCATION: ['localhost:11211'] + default: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + ignore_exc: true + no_delay: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_default + LOCATION: ['localhost:11211'] + general: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + ignore_exc: true + no_delay: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_general + LOCATION: ['localhost:11211'] + mongo_metadata_inheritance: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + ignore_exc: true + no_delay: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_mongo_metadata_inheritance + LOCATION: ['localhost:11211'] + staticfiles: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + ignore_exc: true + no_delay: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_static_files + LOCATION: ['localhost:11211'] +CELERY_ALWAYS_EAGER: true +CELERY_BROKER_HOSTNAME: localhost +CELERY_BROKER_PASSWORD: celery +CELERY_BROKER_TRANSPORT: amqp +CELERY_BROKER_USER: celery +CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION: false +CERT_QUEUE: certificates +CMS_BASE: localhost:8031 +CODE_JAIL: + limits: {REALTIME: 3, VMEM: 0} +COMMENTS_SERVICE_KEY: password +COMMENTS_SERVICE_URL: http://localhost:4567 +CONTACT_EMAIL: info@example.com +CONTENTSTORE: + DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [localhost] + port: 27017 + ENGINE: xmodule.contentstore.mongo.MongoContentStore + OPTIONS: + db: test + host: [localhost] + port: 27017 +DATABASES: + default: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: edxtest, PASSWORD: '', + PORT: '3306', USER: root} + student_module_history: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: student_module_history_test, + PASSWORD: '', PORT: '3306', USER: root} +DEFAULT_FEEDBACK_EMAIL: feedback@example.com +DEFAULT_FROM_EMAIL: registration@example.com +DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, + type: s3fs} +DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [localhost] + port: 27017 +EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend +FEATURES: {CERTIFICATES_HTML_VIEW: true, CUSTOM_COURSES_EDX: true, + ENABLE_CONTENT_LIBRARIES: true, ENABLE_DISCUSSION_SERVICE: true, ENABLE_EXTENDED_COURSE_DETAILS: true, + ENABLE_GRADE_DOWNLOADS: true, ENABLE_SPECIAL_EXAMS: true, ENTRANCE_EXAMS: true, + MILESTONES_APP: true, PREVIEW_LMS_BASE: 'preview.localhost:8003', SHOW_HEADER_LANGUAGE_SELECTOR: true} +GITHUB_REPO_ROOT: '** OVERRIDDEN **' +GRADES_DOWNLOAD: {BUCKET: edx-grades, ROOT_PATH: /tmp/edx-s3/grades, STORAGE_TYPE: localfs} +JWT_AUTH: {JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": + "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', + JWT_SECRET_KEY: super-secret-key} +LMS_BASE: localhost:8003 +LMS_ROOT_URL: http://localhost:8003 +LOCAL_LOGLEVEL: INFO +LOGGING_ENV: sandbox +LOG_DIR: '** OVERRIDDEN **' +MEDIA_URL: /media/ +MKTG_URL_LINK_MAP: {} +MODULESTORE: + default: + ENGINE: xmodule.modulestore.mixed.MixedModuleStore + OPTIONS: + mappings: {} + stores: + - DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [localhost] + port: 27017 + ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore + NAME: draft + OPTIONS: + collection: modulestore + db: test + default_class: xmodule.hidden_block.HiddenBlock + fs_root: '** OVERRIDDEN **' + host: [localhost] + port: 27017 + render_template: common.djangoapps.edxmako.shortcuts.render_to_string + - ENGINE: xmodule.modulestore.xml.XMLModuleStore + NAME: xml + OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} +# We need to test different scenarios, following setting effectively disbale rate limiting +PASSWORD_RESET_IP_RATE: '1/s' +PASSWORD_RESET_EMAIL_RATE: '1/s' +SECRET_KEY: '' +SERVER_EMAIL: devops@example.com +SESSION_COOKIE_DOMAIN: null +SITE_NAME: localhost +SOCIAL_SHARING_SETTINGS: {CUSTOM_COURSE_URLS: true} +STATIC_URL_BASE: /static/ +SYSLOG_SERVER: '' +TECH_SUPPORT_EMAIL: technical@example.com +TIME_ZONE: America/New_York +WIKI_ENABLED: true +XQUEUE_INTERFACE: + basic_auth: [edx, edx] + django_auth: {password: password, username: lms} + url: http://localhost:18040 +ZENDESK_API_KEY: '' +ZENDESK_USER: '' diff --git a/cms/envs/bok_choy_docker.auth.json b/cms/envs/bok_choy_docker.auth.json new file mode 100644 index 000000000000..9a187c05b153 --- /dev/null +++ b/cms/envs/bok_choy_docker.auth.json @@ -0,0 +1,116 @@ +{ + "AWS_ACCESS_KEY_ID": "", + "AWS_SECRET_ACCESS_KEY": "", + "CELERY_BROKER_PASSWORD": "celery", + "CELERY_BROKER_USER": "celery", + "CONTENTSTORE": { + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", + "OPTIONS": { + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + } + }, + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.mysql", + "HOST": "edx.devstack.mysql80", + "NAME": "edxtest", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + }, + "student_module_history": { + "ENGINE": "django.db.backends.mysql", + "HOST": "edx.devstack.mysql80", + "NAME": "student_module_history_test", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + } + }, + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "JWT_AUTH": { + "JWT_SECRET_KEY": "super-secret-key", + "JWT_PUBLIC_SIGNING_JWK_SET": "{\"keys\": [{\"kid\": \"BTZ9HA6K\", \"e\": \"AQAB\", \"kty\": \"RSA\", \"n\": \"o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ\"}]}" + }, + "MODULESTORE": { + "default": { + "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore", + "OPTIONS": { + "mappings": {}, + "stores": [ + { + "NAME": "draft", + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore", + "OPTIONS": { + "collection": "modulestore", + "db": "test", + "default_class": "xmodule.hidden_block.HiddenBlock", + "fs_root": "** OVERRIDDEN **", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017, + "render_template": "common.djangoapps.edxmako.shortcuts.render_to_string" + } + }, + { + "NAME": "xml", + "ENGINE": "xmodule.modulestore.xml.XMLModuleStore", + "OPTIONS": { + "data_dir": "** OVERRIDDEN **", + "default_class": "xmodule.hidden_block.HiddenBlock" + } + } + ] + } + } + }, + "DJFS": { + "type": "s3fs", + "bucket": "test", + "prefix": "test", + "aws_access_key_id": "test", + "aws_secret_access_key": "test" + }, + "SECRET_KEY": "", + "XQUEUE_INTERFACE": { + "basic_auth": [ + "edx", + "edx" + ], + "django_auth": { + "password": "password", + "username": "lms" + }, + "url": "http://localhost:18040" + }, + "ZENDESK_API_KEY": "", + "ZENDESK_USER": "" +} diff --git a/cms/envs/bok_choy_docker.env.json b/cms/envs/bok_choy_docker.env.json new file mode 100644 index 000000000000..90b4ced635ad --- /dev/null +++ b/cms/envs/bok_choy_docker.env.json @@ -0,0 +1,131 @@ +{ + "BUGS_EMAIL": "bugs@example.com", + "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com", + "CACHES": { + "celery": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "integration_celery", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "default": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "sandbox_default", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "general": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "sandbox_general", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "mongo_metadata_inheritance": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "integration_mongo_metadata_inheritance", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "staticfiles": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "OPTIONS": { + "no_delay": true, + "ignore_exc": true, + "use_pooling": true, + "connect_timeout": 0.5 + }, + "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", + "KEY_PREFIX": "integration_static_files", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + } + }, + "CELERY_ALWAYS_EAGER": true, + "CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION": false, + "CELERY_BROKER_HOSTNAME": "localhost", + "CELERY_BROKER_TRANSPORT": "amqp", + "CERT_QUEUE": "certificates", + "CMS_BASE": "** OVERRIDDEN **", + "CODE_JAIL": { + "limits": { + "REALTIME": 3, + "VMEM": 0 + } + }, + "COMMENTS_SERVICE_KEY": "password", + "COMMENTS_SERVICE_URL": "http://edx.devstack.studio:4567", + "CONTACT_EMAIL": "info@example.com", + "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com", + "DEFAULT_FROM_EMAIL": "registration@example.com", + "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", + "SOCIAL_SHARING_SETTINGS": { + "CUSTOM_COURSE_URLS": true + }, + "FEATURES": { + "CERTIFICATES_HTML_VIEW": true, + "ENABLE_DISCUSSION_SERVICE": true, + "ENABLE_GRADE_DOWNLOADS": true, + "ENTRANCE_EXAMS": true, + "MILESTONES_APP": true, + "PREVIEW_LMS_BASE": "preview.localhost:8003", + "ENABLE_CONTENT_LIBRARIES": true, + "ENABLE_SPECIAL_EXAMS": true, + "SHOW_HEADER_LANGUAGE_SELECTOR": true, + "ENABLE_EXTENDED_COURSE_DETAILS": true, + "CUSTOM_COURSES_EDX": true + }, + "GITHUB_REPO_ROOT": "** OVERRIDDEN **", + "GRADES_DOWNLOAD": { + "BUCKET": "edx-grades", + "ROOT_PATH": "/tmp/edx-s3/grades", + "STORAGE_TYPE": "localfs" + }, + "LMS_BASE": "** OVERRIDDEN **", + "LMS_ROOT_URL": "** OVERRIDDEN **", + "LOCAL_LOGLEVEL": "INFO", + "LOGGING_ENV": "sandbox", + "LOG_DIR": "** OVERRIDDEN **", + "MEDIA_URL": "/media/", + "MKTG_URL_LINK_MAP": {}, + "SERVER_EMAIL": "devops@example.com", + "SESSION_COOKIE_DOMAIN": null, + "SITE_NAME": "localhost", + "STATIC_URL_BASE": "/static/", + "SYSLOG_SERVER": "", + "TECH_SUPPORT_EMAIL": "technical@example.com", + "TIME_ZONE": "America/New_York", + "WIKI_ENABLED": true, +} diff --git a/cms/envs/bok_choy_docker.py b/cms/envs/bok_choy_docker.py new file mode 100644 index 000000000000..878537de008d --- /dev/null +++ b/cms/envs/bok_choy_docker.py @@ -0,0 +1,26 @@ +""" +Settings for Bok Choy tests that are used when running Studio in Docker-based devstack. +""" + +# noinspection PyUnresolvedReferences +from .bok_choy import * # pylint: disable=wildcard-import + +CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_CMS_PORT', 8031)) +LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_LMS_PORT', 8003)) +LMS_ROOT_URL = f'http://{LMS_BASE}' +LOGIN_REDIRECT_WHITELIST = [CMS_BASE] + +COMMENTS_SERVICE_URL = 'http://{}:4567'.format(os.environ['BOK_CHOY_HOSTNAME']) +EDXNOTES_PUBLIC_API = 'http://{}:8042/api/v1'.format(os.environ['BOK_CHOY_HOSTNAME']) + +# Docker does not support the syslog socket at /dev/log. Rely on the console. +LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = { + 'class': 'logging.NullHandler', +} + +LOGGING['loggers']['tracking']['handlers'] = ['console'] + +# Point the URL used to test YouTube availability to our stub YouTube server +BOK_CHOY_HOST = os.environ['BOK_CHOY_HOSTNAME'] +YOUTUBE['API'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/get_youtube_api/" +YOUTUBE['METADATA_URL'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/test_youtube/" diff --git a/cms/envs/bok_choy_docker.yml b/cms/envs/bok_choy_docker.yml new file mode 100644 index 000000000000..2b18b5be11c7 --- /dev/null +++ b/cms/envs/bok_choy_docker.yml @@ -0,0 +1,152 @@ +# ingested bok_choy_docker.env.json +# ingested bok_choy_docker.auth.json +AWS_ACCESS_KEY_ID: '' +AWS_SECRET_ACCESS_KEY: '' +BUGS_EMAIL: bugs@example.com +BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com +CACHES: + celery: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_celery + LOCATION: ['edx.devstack.memcached:11211'] + default: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_default + LOCATION: ['edx.devstack.memcached:11211'] + general: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_general + LOCATION: ['edx.devstack.memcached:11211'] + mongo_metadata_inheritance: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_mongo_metadata_inheritance + LOCATION: ['edx.devstack.memcached:11211'] + staticfiles: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_static_files + LOCATION: ['edx.devstack.memcached:11211'] +CELERY_ALWAYS_EAGER: true +CELERY_BROKER_HOSTNAME: localhost +CELERY_BROKER_PASSWORD: celery +CELERY_BROKER_TRANSPORT: amqp +CELERY_BROKER_USER: celery +CERT_QUEUE: certificates +CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION: false +CMS_BASE: '** OVERRIDDEN **' +CODE_JAIL: + limits: {REALTIME: 3, VMEM: 0} +COMMENTS_SERVICE_KEY: password +COMMENTS_SERVICE_URL: http://edx.devstack.studio:4567 +CONTACT_EMAIL: info@example.com +CONTENTSTORE: + DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [edx.devstack.mongo] + port: 27017 + ENGINE: xmodule.contentstore.mongo.MongoContentStore + OPTIONS: + db: test + host: [edx.devstack.mongo] + port: 27017 +DATABASES: + default: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql80, NAME: edxtest, + PASSWORD: '', PORT: '3306', USER: root} + student_module_history: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql80, + NAME: student_module_history_test, PASSWORD: '', PORT: '3306', USER: root} +DEFAULT_FEEDBACK_EMAIL: feedback@example.com +DEFAULT_FROM_EMAIL: registration@example.com +DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, + type: s3fs} +DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [edx.devstack.mongo] + port: 27017 +EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend +FEATURES: {CERTIFICATES_HTML_VIEW: true, CUSTOM_COURSES_EDX: true, + ENABLE_CONTENT_LIBRARIES: true, ENABLE_DISCUSSION_SERVICE: true, ENABLE_EXTENDED_COURSE_DETAILS: true, + ENABLE_GRADE_DOWNLOADS: true, ENABLE_SPECIAL_EXAMS: true, ENTRANCE_EXAMS: true, + MILESTONES_APP: true, PREVIEW_LMS_BASE: 'preview.localhost:8003', SHOW_HEADER_LANGUAGE_SELECTOR: true} +GITHUB_REPO_ROOT: '** OVERRIDDEN **' +GRADES_DOWNLOAD: {BUCKET: edx-grades, ROOT_PATH: /tmp/edx-s3/grades, STORAGE_TYPE: localfs} +JWT_AUTH: {JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": + "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', + JWT_SECRET_KEY: super-secret-key} +LMS_BASE: '** OVERRIDDEN **' +LMS_ROOT_URL: '** OVERRIDDEN **' +LOCAL_LOGLEVEL: INFO +LOGGING_ENV: sandbox +LOG_DIR: '** OVERRIDDEN **' +MEDIA_URL: /media/ +MKTG_URL_LINK_MAP: {} +MODULESTORE: + default: + ENGINE: xmodule.modulestore.mixed.MixedModuleStore + OPTIONS: + mappings: {} + stores: + - DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [edx.devstack.mongo] + port: 27017 + ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore + NAME: draft + OPTIONS: + collection: modulestore + db: test + default_class: xmodule.hidden_block.HiddenBlock + fs_root: '** OVERRIDDEN **' + host: [edx.devstack.mongo] + port: 27017 + render_template: common.djangoapps.edxmako.shortcuts.render_to_string + - ENGINE: xmodule.modulestore.xml.XMLModuleStore + NAME: xml + OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} +SECRET_KEY: '' +SERVER_EMAIL: devops@example.com +SESSION_COOKIE_DOMAIN: null +SITE_NAME: localhost +SOCIAL_SHARING_SETTINGS: {CUSTOM_COURSE_URLS: true} +STATIC_URL_BASE: /static/ +SYSLOG_SERVER: '' +TECH_SUPPORT_EMAIL: technical@example.com +TIME_ZONE: America/New_York +WIKI_ENABLED: true +XQUEUE_INTERFACE: + basic_auth: [edx, edx] + django_auth: {password: password, username: lms} + url: http://localhost:18040 +ZENDESK_API_KEY: '' +ZENDESK_USER: '' diff --git a/cms/envs/test_static_optimized.py b/cms/envs/test_static_optimized.py index c92d9a7262ce..61738c92548b 100644 --- a/cms/envs/test_static_optimized.py +++ b/cms/envs/test_static_optimized.py @@ -1,6 +1,10 @@ """ Settings used when generating static assets for use in tests. +For example, Bok Choy uses two different settings files: +1. test_static_optimized is used when invoking collectstatic +2. bok_choy is used when running CMS and LMS + Note: it isn't possible to have a single settings file, because Django doesn't support both generating static assets to a directory and also serving static from the same directory. diff --git a/common/djangoapps/terrain/stubs/lti.py b/common/djangoapps/terrain/stubs/lti.py index 46535abb9f80..c406da5e6ed3 100644 --- a/common/djangoapps/terrain/stubs/lti.py +++ b/common/djangoapps/terrain/stubs/lti.py @@ -13,6 +13,7 @@ import base64 import hashlib import logging +import os import textwrap from unittest import mock from uuid import uuid4 @@ -77,7 +78,7 @@ def do_POST(self): 'callback_url': self.post_dict.get('lis_outcome_service_url').replace('https', 'http'), 'sourcedId': self.post_dict.get('lis_result_sourcedid') } - host = self.server.server_address[0] + host = os.environ.get('BOK_CHOY_HOSTNAME', self.server.server_address[0]) submit_url = f'//{host}:{self.server.server_address[1]}' content = self._create_content(status_message, submit_url) self.send_response(200, content) @@ -295,7 +296,7 @@ def _check_oauth_signature(self, params, client_signature): """ client_secret = str(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET)) - host = '127.0.0.1' + host = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') port = self.server.server_address[1] lti_base = self.DEFAULT_LTI_ADDRESS.format(host=host, port=port) lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT) diff --git a/common/static/common/js/karma.common.conf.js b/common/static/common/js/karma.common.conf.js index 09f6a21848c3..7b92442aaf6b 100644 --- a/common/static/common/js/karma.common.conf.js +++ b/common/static/common/js/karma.common.conf.js @@ -281,6 +281,15 @@ function getBaseConfig(config, useRequireJs) { 'framework:custom': ['factory', initFrameworks] }; + if (process.env.hasOwnProperty('BOK_CHOY_HOSTNAME')) { + hostname = process.env.BOK_CHOY_HOSTNAME; + if (hostname === 'edx.devstack.lms') { + port = 19876; + } else { + port = 19877; + } + } + initFrameworks.$inject = ['config.files']; return { @@ -376,7 +385,7 @@ function getBaseConfig(config, useRequireJs) { } } }, - + // Continuous Integration mode // if true, Karma captures browsers, runs the tests and exits singleRun: config.singleRun, diff --git a/docs/decisions/0004-managing-django-settings.rst b/docs/decisions/0004-managing-django-settings.rst index 6da93a76163e..97c15b6cf4b5 100644 --- a/docs/decisions/0004-managing-django-settings.rst +++ b/docs/decisions/0004-managing-django-settings.rst @@ -59,6 +59,7 @@ and for all environments to use __init__.py to load their settings. The following files should be obviated by this change: * bok_choy_docker.py +* bok_choy.py * devstack_docker.py * devstack_optimized.py * devstack.py diff --git a/lms/envs/bok_choy.py b/lms/envs/bok_choy.py new file mode 100644 index 000000000000..9a39d4d8bb72 --- /dev/null +++ b/lms/envs/bok_choy.py @@ -0,0 +1,154 @@ +""" +Settings for Bok Choy tests that are used when running LMS. + +Bok Choy uses two different settings files: +1. test_static_optimized is used when invoking collectstatic +2. bok_choy is used when running the tests + +Note: it isn't possible to have a single settings file, because Django doesn't +support both generating static assets to a directory and also serving static +from the same directory. +""" + + +# Silence noisy logs +import logging +import os +from tempfile import mkdtemp + +from django.utils.translation import gettext_lazy +from path import Path as path + +from openedx.core.release import RELEASE_LINE +from xmodule.modulestore.modulestore_settings import update_module_store_settings # lint-amnesty, pylint: disable=wrong-import-order + +CONFIG_ROOT = path(__file__).abspath().dirname() +TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" + +########################## Prod-like settings ################################### +# These should be as close as possible to the settings we use in production. +# As in prod, we read in environment and auth variables from JSON files. +# Unlike in prod, we use the JSON files stored in this repo. +# This is a convenience for ensuring (a) that we can consistently find the files +# and (b) that the files are the same in Jenkins as in local dev. +os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy' +os.environ['LMS_CFG'] = str.format("{config_root}/{service_variant}.yml", + config_root=CONFIG_ROOT, service_variant=os.environ['SERVICE_VARIANT']) +os.environ['REVISION_CFG'] = f"{CONFIG_ROOT}/revisions.yml" + +from .production import * # pylint: disable=wildcard-import, unused-wildcard-import, wrong-import-position + + +######################### Testing overrides #################################### + +# Redirect to the test_root folder within the repo +GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath() +LOG_DIR = (TEST_ROOT / "log").abspath() + +# Configure modulestore to use the test folder within the repo +update_module_store_settings( + MODULESTORE, + module_store_options={ + 'fs_root': (TEST_ROOT / "data").abspath(), + }, + xml_store_options={ + 'data_dir': (TEST_ROOT / "data").abspath(), + }, + default_store=os.environ.get('DEFAULT_STORE', 'draft'), +) + +PLATFORM_NAME = gettext_lazy("édX") +PLATFORM_DESCRIPTION = gettext_lazy("Open édX Platform") + +############################ STATIC FILES ############################# + +# Serve static files at /static directly from the staticfiles directory under test root +# Note: optimized files for testing are generated with settings from test_static_optimized +STATIC_URL = "/static/" +STATICFILES_FINDERS = ['django.contrib.staticfiles.finders.FileSystemFinder'] +STATICFILES_DIRS = [ + (TEST_ROOT / "staticfiles" / "lms").abspath(), +] + +DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' +MEDIA_ROOT = TEST_ROOT / "uploads" + +# Webpack loader must use webpack output setting +WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = TEST_ROOT / "staticfiles" / "lms" / "webpack-stats.json" + +# Don't use compression during tests +PIPELINE['JS_COMPRESSOR'] = None + +###################### Grades ###################### +GRADES_DOWNLOAD = { + 'STORAGE_TYPE': 'localfs', + 'BUCKET': 'edx-grades', + 'ROOT_PATH': os.path.join(mkdtemp(), 'edx-s3', 'grades'), +} + + +LOG_OVERRIDES = [ + ('track.middleware', logging.CRITICAL), + ('common.djangoapps.edxmako.shortcuts', logging.ERROR), + ('edx.discussion', logging.CRITICAL), +] +for log_name, log_level in LOG_OVERRIDES: + logging.getLogger(log_name).setLevel(log_level) + + +YOUTUBE_HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') +# Point the URL used to test YouTube availability to our stub YouTube server +YOUTUBE_PORT = 9080 +YOUTUBE['TEST_TIMEOUT'] = 5000 +YOUTUBE['API'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/get_youtube_api/" +YOUTUBE['METADATA_URL'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/test_youtube/" + +############################# SECURITY SETTINGS ################################ +# Default to advanced security in common.py, so tests can reset here to use +# a simpler security model + +# Path at which to store the mock index +MOCK_SEARCH_BACKING_FILE = ( + TEST_ROOT / "index_file.dat" +).abspath() + +# Verify student settings +VERIFY_STUDENT["SOFTWARE_SECURE"] = { + "API_ACCESS_KEY": "BBBBBBBBBBBBBBBBBBBB", + "API_SECRET_KEY": "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", +} + +# Set dummy values for profile image settings. +PROFILE_IMAGE_BACKEND = { + 'class': 'openedx.core.storage.OverwriteStorage', + 'options': { + 'location': os.path.join(MEDIA_ROOT, 'profile-images/'), + 'base_url': os.path.join(MEDIA_URL, 'profile-images/'), + }, +} + +LMS_ROOT_URL = "http://localhost:{}".format(os.environ.get('BOK_CHOY_LMS_PORT', 8003)) +CMS_BASE = "localhost:{}".format(os.environ.get('BOK_CHOY_CMS_PORT', 8031)) +LOGIN_REDIRECT_WHITELIST = [CMS_BASE] + +INSTALLED_APPS.append('openedx.testing.coverage_context_listener') + +if RELEASE_LINE == "master": + # On master, acceptance tests use edX books, not the default Open edX books. + HELP_TOKENS_BOOKS = { + 'learner': 'https://edx.readthedocs.io/projects/edx-guide-for-students', + 'course_author': 'https://edx.readthedocs.io/projects/edx-partner-course-staff', + } + +# API access management +API_ACCESS_MANAGER_EMAIL = 'api-access@example.com' +API_ACCESS_FROM_EMAIL = 'api-requests@example.com' +API_DOCUMENTATION_URL = 'https://course-catalog-api-guide.readthedocs.io/en/latest/' +AUTH_DOCUMENTATION_URL = 'https://course-catalog-api-guide.readthedocs.io/en/latest/authentication/index.html' + +##################################################################### +# Lastly, see if the developer has any local overrides. +try: + from .private import * # pylint: disable=wildcard-import +except ImportError: + pass diff --git a/lms/envs/bok_choy.yml b/lms/envs/bok_choy.yml new file mode 100644 index 000000000000..a14038b3b2ec --- /dev/null +++ b/lms/envs/bok_choy.yml @@ -0,0 +1,276 @@ +# ingested edx-platform/lms/envs/bok_choy.auth.json +# ingested edx-platform/lms/envs/bok_choy.env.json +ACTIVATION_EMAIL_SUPPORT_LINK: https://support.example.com/activation-email-help.html +ANALYTICS_DASHBOARD_URL: '' +AWS_ACCESS_KEY_ID: '' +AWS_SECRET_ACCESS_KEY: '' +BUGS_EMAIL: bugs@example.com +BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com +BLOCK_STRUCTURES_SETTINGS: + # We have CELERY_ALWAYS_EAGER set to True, so there's no asynchronous + # code running and the celery routing is unimportant. + # It does not make sense to retry. + TASK_MAX_RETRIES: 0 + # course publish task delay is irrelevant is because the task is run synchronously + COURSE_PUBLISH_TASK_DELAY: 0 + # retry delay is irrelevent because we never retry + TASK_DEFAULT_RETRY_DELAY: 0 + +CACHES: + celery: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_celery + LOCATION: ['localhost:11211'] + default: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_default + LOCATION: ['localhost:11211'] + general: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_general + LOCATION: ['localhost:11211'] + mongo_metadata_inheritance: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_mongo_metadata_inheritance + LOCATION: ['localhost:11211'] + staticfiles: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_static_files + LOCATION: ['localhost:11211'] + +# Capture the console log via template includes, until webdriver supports log capture again +CAPTURE_CONSOLE_LOG: True + +CELERY_BROKER_HOSTNAME: localhost +CELERY_BROKER_PASSWORD: celery +CELERY_BROKER_TRANSPORT: amqp +CELERY_BROKER_USER: celery +CELERY_ALWAYS_EAGER: True +CELERY_RESULT_BACKEND: 'django-cache' +CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION: False + +CERT_QUEUE: certificates +CMS_BASE: localhost:8031 +CODE_JAIL: + limits: {REALTIME: 3, VMEM: 0} +COMMENTS_SERVICE_KEY: password +COMMENTS_SERVICE_URL: http://localhost:4567 +COMPLETION_BY_VIEWING_DELAY_MS: 1000 +CONTACT_EMAIL: info@example.com +CONTENTSTORE: + DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [localhost] + port: 27017 + ENGINE: xmodule.contentstore.mongo.MongoContentStore + OPTIONS: + db: test + host: [localhost] + port: 27017 +DATABASES: + default: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: edxtest, PASSWORD: '', + PORT: '3306', USER: root} + student_module_history: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: student_module_history_test, + PASSWORD: '', PORT: '3306', USER: root} +DEFAULT_FEEDBACK_EMAIL: feedback@example.com +DEFAULT_FROM_EMAIL: registration@example.com + +# Enable debug so that static assets are served by Django +DEBUG: True +DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, + type: s3fs} +DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [localhost] + port: 27017 +# Configure the LMS to use our stub eCommerce implementation +ECOMMERCE_API_URL: 'http://localhost:8043/api/v2/' +# Configure the LMS to use our stub EdxNotes implementation +EDXNOTES_PUBLIC_API: 'http://localhost:8042/api/v1' +EDXNOTES_INTERNAL_API: 'http://localhost:8042/api/v1' + +EDXNOTES_CONNECT_TIMEOUT: 10 # time in seconds +EDXNOTES_READ_TIMEOUT: 10 # time in seconds + +NOTES_DISABLED_TABS: [] + +EMAIL_BACKEND: django.core.mail.backends.dummy.EmailBackend +EVENT_TRACKING_BACKENDS: + mongo: + ENGINE: eventtracking.backends.mongodb.MongoBackend + OPTIONS: {collection: events, database: test} +FEATURES: + ALLOW_AUTOMATED_SIGNUPS: true + AUTOMATIC_AUTH_FOR_TESTING: true + AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING: true + CERTIFICATES_HTML_VIEW: true + CERTIFICATES_INSTRUCTOR_GENERATION: true + CUSTOM_COURSES_EDX: true, + ENABLE_COURSE_DISCOVERY: true + ENABLE_DISCUSSION_SERVICE: true + ENABLE_GRADE_DOWNLOADS: true + ENABLE_SPECIAL_EXAMS: true + ENABLE_THIRD_PARTY_AUTH: true + ENABLE_VERIFIED_CERTIFICATES: true + EXPOSE_CACHE_PROGRAMS_ENDPOINT: true + MODE_CREATION_FOR_TESTING: true + PREVIEW_LMS_BASE: 'preview.localhost:8003' + RESTRICT_AUTOMATIC_AUTH: false + SHOW_HEADER_LANGUAGE_SELECTOR: true + ENABLE_MAX_FAILED_LOGIN_ATTEMPTS: False + SQUELCH_PII_IN_LOGS: False + PREVENT_CONCURRENT_LOGINS: False + ENABLE_MOBILE_REST_API: True # Show video bumper in LMS + ENABLE_VIDEO_BUMPER: True # Show video bumper in LMS + SHOW_BUMPER_PERIODICITY: 1 + # Enable courseware search for tests + ENABLE_COURSEWARE_SEARCH: True + # Enable dashboard search for tests + ENABLE_DASHBOARD_SEARCH: True + # discussion home panel, which includes a subscription on/off setting for discussion digest emails. + ENABLE_DISCUSSION_HOME_PANEL: True + ENABLE_LTI_PROVIDER: True + # Enable milestones app + MILESTONES_APP: True + # Enable oauth authentication, which we test. + ENABLE_OAUTH2_PROVIDER: True + OAUTH_ENFORCE_SECURE: False + ENABLE_PREREQUISITE_COURSES: True + ENABLE_COURSE_DISCOVERY: True + ENABLE_EDXNOTES: True + ENABLE_TEAMS: True + LICENSING: True + # Use the auto_auth workflow for creating users and logging them in + AUTOMATIC_AUTH_FOR_TESTING: True + RESTRICT_AUTOMATIC_AUTH: False + # Open up endpoint for faking Software Secure responses + ENABLE_SOFTWARE_SECURE_FAKE: True + # Disable instructor dash buttons for downloading course data when enrollment exceeds this number + MAX_ENROLLMENT_INSTR_BUTTONS: 4 + ENABLE_ENROLLMENT_TRACK_USER_PARTITION: True + ENTRANCE_EXAMS: True + ENABLE_SPECIAL_EXAMS: True + + +GITHUB_REPO_ROOT: '** OVERRIDDEN **' +JWT_AUTH: {JWT_PRIVATE_SIGNING_JWK: '{"e": "AQAB", "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", + "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", + "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", + "kid": "BTZ9HA6K", "kty": "RSA"}', JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": + "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', + JWT_SECRET_KEY: super-secret-key} +LMS_BASE: localhost:8003 +LMS_ROOT_URL: http://localhost:8003 +LOCAL_LOGLEVEL: INFO +LOGGING_ENV: sandbox +LOG_DIR: '** OVERRIDDEN **' +MEDIA_URL: /media/ +MKTG_URL_LINK_MAP: {ABOUT: about, BLOG: blog, CAREERS: careers, CONTACT: contact, + COURSES: courses, DONATE: donate, HELP_CENTER: help-center, HONOR: honor, NEWS: news, + PRESS: press, PRIVACY: privacy, ROOT: root, SITEMAP.XML: sitemap_xml, TOS: tos, + WHAT_IS_VERIFIED_CERT: verified-certificate} +MODULESTORE: + default: + ENGINE: xmodule.modulestore.mixed.MixedModuleStore + OPTIONS: + mappings: {} + stores: + - DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [localhost] + port: 27017 + ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore + NAME: draft + OPTIONS: + collection: modulestore + db: test + default_class: xmodule.hidden_block.HiddenBlock + fs_root: '** OVERRIDDEN **' + host: [localhost] + port: 27017 + render_template: common.djangoapps.edxmako.shortcuts.render_to_string + - ENGINE: xmodule.modulestore.xml.XMLModuleStore + NAME: xml + OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} +# We need to test different scenarios, following setting effectively disbale rate limiting +PASSWORD_RESET_IP_RATE: '1/s' +PASSWORD_RESET_EMAIL_RATE: '1/s' +PASSWORD_RESET_SUPPORT_LINK: https://support.example.com/password-reset-help.html +REGISTRATION_EXTENSION_FORM: openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm +REGISTRATION_EXTRA_FIELDS: {city: hidden, country: required, gender: optional, goals: optional, + honor_code: required, level_of_education: optional, mailing_address: optional, terms_of_service: hidden, + year_of_birth: optional} +# Use MockSearchEngine as the search engine for test scenario +SEARCH_ENGINE: "search.tests.mock_search_engine.MockSearchEngine" +# this secret key should be the same as cms/envs/bok_choy.py's +SECRET_KEY: "very_secret_bok_choy_key" + +SERVER_EMAIL: devops@example.com +SESSION_COOKIE_DOMAIN: null +SITE_NAME: localhost:8003 +SOCIAL_SHARING_SETTINGS: {CERTIFICATE_FACEBOOK: true, CERTIFICATE_FACEBOOK_TEXT: 'Testing + facebook feature:', CUSTOM_COURSE_URLS: true, DASHBOARD_FACEBOOK: true, DASHBOARD_TWITTER: true, + DASHBOARD_TWITTER_TEXT: 'Testing feature:'} +STATIC_URL_BASE: /static/ +SUPPORT_SITE_LINK: https://support.example.com +SYSLOG_SERVER: '' +TECH_SUPPORT_EMAIL: technical@example.com +THIRD_PARTY_AUTH_BACKENDS: [social_core.backends.google.GoogleOAuth2, social_core.backends.linkedin.LinkedinOAuth2, + social_core.backends.facebook.FacebookOAuth2, common.djangoapps.third_party_auth.dummy.DummyBackend, + common.djangoapps.third_party_auth.saml.SAMLAuthBackend] +THIRD_PARTY_AUTH: + Google: + SOCIAL_AUTH_GOOGLE_OAUTH2_KEY": "test" + SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET": "test" + Facebook: + SOCIAL_AUTH_FACEBOOK_KEY": "test" + SOCIAL_AUTH_FACEBOOK_SECRET": "test" +TIME_ZONE: America/New_York +TRACKING_BACKENDS: + mongo: + ENGINE: common.djangoapps.track.backends.mongodb.MongoBackend + OPTIONS: {collection: events, database: test} +WIKI_ENABLED: true +WAFFLE_OVERRIDE: True +XQUEUE_INTERFACE: + basic_auth: [edx, edx] + django_auth: {password: password, username: lms} + # Configure the LMS to use our stub XQueue implementation + url: 'http://localhost:8040' + +ZENDESK_API_KEY: '' +ZENDESK_USER: '' diff --git a/lms/envs/bok_choy_docker.py b/lms/envs/bok_choy_docker.py new file mode 100644 index 000000000000..991b22e37361 --- /dev/null +++ b/lms/envs/bok_choy_docker.py @@ -0,0 +1,27 @@ +""" +Settings for Bok Choy tests that are used when running Studio in Docker-based devstack. +""" + +# noinspection PyUnresolvedReferences +from .bok_choy import * # pylint: disable=wildcard-import + +CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_CMS_PORT', 8031)) +LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_LMS_PORT', 8003)) +LMS_ROOT_URL = f'http://{LMS_BASE}' +LOGIN_REDIRECT_WHITELIST = [CMS_BASE] +SITE_NAME = LMS_BASE + +COMMENTS_SERVICE_URL = 'http://{}:4567'.format(os.environ['BOK_CHOY_HOSTNAME']) +EDXNOTES_PUBLIC_API = 'http://{}:8042/api/v1'.format(os.environ['BOK_CHOY_HOSTNAME']) + +# Docker does not support the syslog socket at /dev/log. Rely on the console. +LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = { + 'class': 'logging.NullHandler', +} + +LOGGING['loggers']['tracking']['handlers'] = ['console'] + +# Point the URL used to test YouTube availability to our stub YouTube server +BOK_CHOY_HOST = os.environ['BOK_CHOY_HOSTNAME'] +YOUTUBE['API'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/get_youtube_api/" +YOUTUBE['METADATA_URL'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/test_youtube/" diff --git a/lms/envs/bok_choy_docker.yml b/lms/envs/bok_choy_docker.yml new file mode 100644 index 000000000000..9ed43eb70b68 --- /dev/null +++ b/lms/envs/bok_choy_docker.yml @@ -0,0 +1,188 @@ +# ingested edx-platform/lms/envs/bok_choy_docker.auth.json +# ingested edx-platform/lms/envs/bok_choy_docker.env.json +ACTIVATION_EMAIL_SUPPORT_LINK: https://support.example.com/activation-email-help.html +ANALYTICS_DASHBOARD_URL: '' +AWS_ACCESS_KEY_ID: '' +AWS_SECRET_ACCESS_KEY: '' +BUGS_EMAIL: bugs@example.com +BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com +CACHES: + celery: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_celery + LOCATION: ['edx.devstack.memcached:11211'] + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + default: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_default + LOCATION: ['edx.devstack.memcached:11211'] + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + general: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: sandbox_general + LOCATION: ['edx.devstack.memcached:11211'] + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + mongo_metadata_inheritance: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_mongo_metadata_inheritance + LOCATION: ['edx.devstack.memcached:11211'] + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 + staticfiles: + BACKEND: django.core.cache.backends.memcached.PyMemcacheCache + KEY_FUNCTION: common.djangoapps.util.memcache.safe_key + KEY_PREFIX: integration_static_files + LOCATION: ['edx.devstack.memcached:11211'] + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true + connect_timeout: 0.5 +CELERY_BROKER_HOSTNAME: localhost +CELERY_BROKER_PASSWORD: celery +CELERY_BROKER_TRANSPORT: amqp +CELERY_BROKER_USER: celery +CERT_QUEUE: certificates +CMS_BASE: '** OVERRIDDEN **' +CODE_JAIL: + limits: {REALTIME: 3, VMEM: 0} +COMMENTS_SERVICE_KEY: password +COMMENTS_SERVICE_URL: http://edx.devstack.lms:4567 +CONTACT_EMAIL: info@example.com +CONTENTSTORE: + DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [edx.devstack.mongo] + port: 27017 + ENGINE: xmodule.contentstore.mongo.MongoContentStore + OPTIONS: + db: test + host: [edx.devstack.mongo] + port: 27017 +DATABASES: + default: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql80, NAME: edxtest, + PASSWORD: '', PORT: '3306', USER: root} + student_module_history: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql80, + NAME: student_module_history_test, PASSWORD: '', PORT: '3306', USER: root} +DEFAULT_FEEDBACK_EMAIL: feedback@example.com +DEFAULT_FROM_EMAIL: registration@example.com +DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, + type: s3fs} +DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [edx.devstack.mongo] + port: 27017 +EMAIL_BACKEND: django.core.mail.backends.dummy.EmailBackend +EVENT_TRACKING_BACKENDS: + mongo: + ENGINE: eventtracking.backends.mongodb.MongoBackend + OPTIONS: + collection: events + database: test + host: [edx.devstack.mongo] + port: 27017 +FEATURES: {ALLOW_AUTOMATED_SIGNUPS: true, AUTOMATIC_AUTH_FOR_TESTING: true, + AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING: true, CERTIFICATES_HTML_VIEW: true, + CERTIFICATES_INSTRUCTOR_GENERATION: true, CUSTOM_COURSES_EDX: true, + ENABLE_COURSE_DISCOVERY: true, ENABLE_DISCUSSION_SERVICE: true, ENABLE_GRADE_DOWNLOADS: true, + ENABLE_SPECIAL_EXAMS: true, ENABLE_THIRD_PARTY_AUTH: true, + ENABLE_VERIFIED_CERTIFICATES: true, EXPOSE_CACHE_PROGRAMS_ENDPOINT: true, MODE_CREATION_FOR_TESTING: true, + PREVIEW_LMS_BASE: 'preview.localhost:8003', RESTRICT_AUTOMATIC_AUTH: false, SHOW_HEADER_LANGUAGE_SELECTOR: true} +GITHUB_REPO_ROOT: '** OVERRIDDEN **' +JWT_AUTH: {JWT_PRIVATE_SIGNING_JWK: '{"e": "AQAB", "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", + "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", + "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", + "kid": "BTZ9HA6K", "kty": "RSA"}', JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": + "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', + JWT_SECRET_KEY: super-secret-key} +LMS_BASE: http://edx.devstack.lms:18003 +LMS_ROOT_URL: http://edx.devstack.lms:18003 +LOCAL_LOGLEVEL: INFO +LOGGING_ENV: sandbox +LOG_DIR: '** OVERRIDDEN **' +MEDIA_URL: /media/ +MKTG_URL_LINK_MAP: {ABOUT: about, BLOG: blog, CAREERS: careers, CONTACT: contact, + COURSES: courses, DONATE: donate, HELP_CENTER: help-center, HONOR: honor, NEWS: news, + PRESS: press, PRIVACY: privacy, ROOT: root, SITEMAP.XML: sitemap_xml, TOS: tos, + WHAT_IS_VERIFIED_CERT: verified-certificate} +MODULESTORE: + default: + ENGINE: xmodule.modulestore.mixed.MixedModuleStore + OPTIONS: + mappings: {} + stores: + - DOC_STORE_CONFIG: + collection: modulestore + db: test + host: [edx.devstack.mongo] + port: 27017 + ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore + NAME: draft + OPTIONS: + collection: modulestore + db: test + default_class: xmodule.hidden_block.HiddenBlock + fs_root: '** OVERRIDDEN **' + host: [edx.devstack.mongo] + port: 27017 + render_template: common.djangoapps.edxmako.shortcuts.render_to_string + - ENGINE: xmodule.modulestore.xml.XMLModuleStore + NAME: xml + OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} +PASSWORD_RESET_SUPPORT_LINK: https://support.example.com/password-reset-help.html +REGISTRATION_EXTENSION_FORM: openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm +REGISTRATION_EXTRA_FIELDS: {city: hidden, country: required, gender: optional, goals: optional, + honor_code: required, level_of_education: optional, mailing_address: optional, terms_of_service: hidden, + year_of_birth: optional} +SECRET_KEY: 'bokchoy_docker_secret_key' +SERVER_EMAIL: devops@example.com +SESSION_COOKIE_DOMAIN: null +SITE_NAME: localhost:8003 +SOCIAL_SHARING_SETTINGS: {CERTIFICATE_FACEBOOK: true, CERTIFICATE_FACEBOOK_TEXT: 'Testing + facebook feature:', CUSTOM_COURSE_URLS: true, DASHBOARD_FACEBOOK: true, DASHBOARD_TWITTER: true, + DASHBOARD_TWITTER_TEXT: 'Testing feature:'} +STATIC_URL_BASE: /static/ +SUPPORT_SITE_LINK: https://support.example.com +SYSLOG_SERVER: '' +TECH_SUPPORT_EMAIL: technical@example.com +THIRD_PARTY_AUTH_BACKENDS: [social_core.backends.google.GoogleOAuth2, social_core.backends.linkedin.LinkedinOAuth2, + social_core.backends.facebook.FacebookOAuth2, common.djangoapps.third_party_auth.dummy.DummyBackend, + common.djangoapps.third_party_auth.saml.SAMLAuthBackend] +TIME_ZONE: America/New_York +TRACKING_BACKENDS: + mongo: + ENGINE: common.djangoapps.track.backends.mongodb.MongoBackend + OPTIONS: + collection: events + database: test + host: [edx.devstack.mongo] + port: 27017 +WIKI_ENABLED: true +XQUEUE_INTERFACE: + basic_auth: [edx, edx] + django_auth: {password: password, username: lms} + url: '** OVERRIDDEN **' +ZENDESK_API_KEY: '' +ZENDESK_USER: '' diff --git a/lms/envs/test_static_optimized.py b/lms/envs/test_static_optimized.py index b57276b04036..5c9cf062083f 100644 --- a/lms/envs/test_static_optimized.py +++ b/lms/envs/test_static_optimized.py @@ -1,6 +1,10 @@ """ Settings used when generating static assets for use in tests. +For example, Bok Choy uses two different settings files: +1. test_static_optimized is used when invoking collectstatic +2. bok_choy is used when running CMS and LMS + Note: it isn't possible to have a single settings file, because Django doesn't support both generating static assets to a directory and also serving static from the same directory. diff --git a/openedx/core/djangoapps/util/management/commands/update_fixtures.py b/openedx/core/djangoapps/util/management/commands/update_fixtures.py new file mode 100644 index 000000000000..53e7db56eae4 --- /dev/null +++ b/openedx/core/djangoapps/util/management/commands/update_fixtures.py @@ -0,0 +1,30 @@ +""" +Django management command to update the loaded test fixtures as necessary for +the current test environment. Currently just sets an appropriate domain for +each Site fixture. +""" + + +import os + +from django.contrib.sites.models import Site +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + """ + update_fixtures management command + """ + + help = "Update fixtures to match the current test environment." + + def handle(self, *args, **options): + if 'BOK_CHOY_HOSTNAME' in os.environ: + # Fix the Site fixture domains so third party auth tests work correctly + host = os.environ['BOK_CHOY_HOSTNAME'] + cms_port = os.environ['BOK_CHOY_CMS_PORT'] + lms_port = os.environ['BOK_CHOY_LMS_PORT'] + cms_domain = f'{host}:{cms_port}' + Site.objects.filter(name='cms').update(domain=cms_domain) + lms_domain = f'{host}:{lms_port}' + Site.objects.filter(name='lms').update(domain=lms_domain) diff --git a/openedx/core/djangoapps/util/tests/test_update_fixtures.py b/openedx/core/djangoapps/util/tests/test_update_fixtures.py new file mode 100644 index 000000000000..fb43fa34a88c --- /dev/null +++ b/openedx/core/djangoapps/util/tests/test_update_fixtures.py @@ -0,0 +1,42 @@ +""" # lint-amnesty, pylint: disable=django-not-configured +Tests of the update_fixtures management command for bok-choy test database +initialization. +""" + + +import os + +import pytest +from django.contrib.sites.models import Site +from django.core.management import call_command + + +@pytest.fixture(scope='function') +def sites(db): # lint-amnesty, pylint: disable=unused-argument + Site.objects.create(name='cms', domain='localhost:8031') + Site.objects.create(name='lms', domain='localhost:8003') + + +def test_localhost(db, monkeypatch, sites): # lint-amnesty, pylint: disable=redefined-outer-name, unused-argument + monkeypatch.delitem(os.environ, 'BOK_CHOY_HOSTNAME', raising=False) + call_command('update_fixtures') + assert Site.objects.get(name='cms').domain == 'localhost:8031' + assert Site.objects.get(name='lms').domain == 'localhost:8003' + + +def test_devstack_cms(db, monkeypatch, sites): # lint-amnesty, pylint: disable=redefined-outer-name, unused-argument + monkeypatch.setitem(os.environ, 'BOK_CHOY_HOSTNAME', 'edx.devstack.cms') + monkeypatch.setitem(os.environ, 'BOK_CHOY_CMS_PORT', '18031') + monkeypatch.setitem(os.environ, 'BOK_CHOY_LMS_PORT', '18003') + call_command('update_fixtures') + assert Site.objects.get(name='cms').domain == 'edx.devstack.cms:18031' + assert Site.objects.get(name='lms').domain == 'edx.devstack.cms:18003' + + +def test_devstack_lms(db, monkeypatch, sites): # lint-amnesty, pylint: disable=redefined-outer-name, unused-argument + monkeypatch.setitem(os.environ, 'BOK_CHOY_HOSTNAME', 'edx.devstack.lms') + monkeypatch.setitem(os.environ, 'BOK_CHOY_CMS_PORT', '18031') + monkeypatch.setitem(os.environ, 'BOK_CHOY_LMS_PORT', '18003') + call_command('update_fixtures') + assert Site.objects.get(name='cms').domain == 'edx.devstack.lms:18031' + assert Site.objects.get(name='lms').domain == 'edx.devstack.lms:18003' diff --git a/openedx/features/content_type_gating/tests/test_access.py b/openedx/features/content_type_gating/tests/test_access.py index c42a530e3ccf..c66359fc923a 100644 --- a/openedx/features/content_type_gating/tests/test_access.py +++ b/openedx/features/content_type_gating/tests/test_access.py @@ -1,6 +1,7 @@ """ Test audit user's access to various content based on content-gating features. """ +import os from datetime import datetime, timedelta from unittest.mock import patch, Mock @@ -227,7 +228,8 @@ def setUpClass(cls): graded=False, ) cls.graded_score_weight_blocks[(graded, has_score, weight)] = block - host = '127.0.0.1' + + host = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') metadata_lti_xblock = { 'lti_id': 'correct_lti_id', 'launch_url': 'http://{}:{}/{}'.format(host, '8765', 'correct_lti_endpoint'), diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py new file mode 100644 index 000000000000..cdafa7df8143 --- /dev/null +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -0,0 +1,52 @@ +""" +A pytest plugin that reports test contexts to coverage running in another process. +""" + +import pytest +import requests + +from pavelib.utils.envs import Env + + +class RemoteContextPlugin: + """ + Pytest plugin for reporting pytests contexts to coverage running in another process + """ + def __init__(self, config): + self.config = config + self.active = config.getoption("pytest-contexts") + + def pytest_runtest_setup(self, item): + self.doit(item, "setup") + + def pytest_runtest_teardown(self, item): + self.doit(item, "teardown") + + def pytest_runtest_call(self, item): + self.doit(item, "call") + + def doit(self, item, when): # lint-amnesty, pylint: disable=missing-function-docstring + if self.active: + for cfg in Env.BOK_CHOY_SERVERS.values(): + result = requests.post( + 'http://{host}:{port}/coverage_context/update_context'.format(**cfg), + { + 'context': f"{item.nodeid}|{when}", + } + ) + assert result.status_code == 204 + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") + + +def pytest_addoption(parser): + group = parser.getgroup("coverage") + group.addoption( + "--pytest-remote-contexts", + action="store_true", + dest="pytest-contexts", + help="Capture the pytest contexts that coverage is being captured in in another process", + ) diff --git a/openedx/tests/xblock_integration/xblock_testcase.py b/openedx/tests/xblock_integration/xblock_testcase.py index 7e6d9a9387f9..e9488b3f21f2 100644 --- a/openedx/tests/xblock_integration/xblock_testcase.py +++ b/openedx/tests/xblock_integration/xblock_testcase.py @@ -71,7 +71,7 @@ class XBlockEventTestMixin: 2. assert_event_published verifies that an event of a given search specification was published. - The Mongo event tests in cohorts have nice examplars for + The Mongo/bok_choy event tests in cohorts have nice examplars for how such functionality might look. In the future, we would like to expand both search diff --git a/pavelib/paver_tests/test_utils.py b/pavelib/paver_tests/test_utils.py index ff468ecc9b46..7a157b04ea2c 100644 --- a/pavelib/paver_tests/test_utils.py +++ b/pavelib/paver_tests/test_utils.py @@ -12,6 +12,7 @@ from pavelib.utils.test.utils import MINIMUM_FIREFOX_VERSION, check_firefox_version +@unittest.skipIf(Env.USING_DOCKER, 'Firefox version check works differently under Docker Devstack') class TestUtils(unittest.TestCase): """ Test utils.py under pavelib/utils/test diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py index f925fe166261..a1b14c54455d 100644 --- a/pavelib/prereqs.py +++ b/pavelib/prereqs.py @@ -354,3 +354,13 @@ def install_prereqs(): def log_installed_python_prereqs(): """ Logs output of pip freeze for debugging. """ sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) + + +def print_devstack_warning(): # lint-amnesty, pylint: disable=missing-function-docstring + if Env.USING_DOCKER: # pragma: no cover + print("********************************************************************************") + print("* WARNING: Mac users should run this from both the lms and studio shells") + print("* in docker devstack to avoid startup errors that kill your CPU.") + print("* For more details, see:") + print("* https://github.com/openedx/devstack#docker-is-using-lots-of-cpu-time-when-it-should-be-idle") + print("********************************************************************************") diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index ed2553a42fa2..9e2e2fe5b831 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -61,6 +61,18 @@ class Env: # Python unittest dirs PYTHON_COVERAGERC = REPO_ROOT / ".coveragerc" + # Bok_choy dirs + BOK_CHOY_DIR = REPO_ROOT / "common" / "test" / "acceptance" + BOK_CHOY_LOG_DIR = GEN_LOG_DIR + BOK_CHOY_REPORT_DIR = REPORT_DIR / "bok_choy" + BOK_CHOY_A11Y_REPORT_DIR = REPORT_DIR / "a11y" + BOK_CHOY_COVERAGERC = BOK_CHOY_DIR / ".coveragerc" + BOK_CHOY_A11Y_COVERAGERC = BOK_CHOY_DIR / ".a11ycoveragerc" + BOK_CHOY_A11Y_CUSTOM_RULES_FILE = ( + REPO_ROOT / "node_modules" / "edx-custom-a11y-rules" / + "lib" / "custom_a11y_rules.js" + ) + # Which Python version should be used in xdist workers? PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7") @@ -70,20 +82,94 @@ class Env: # build steps. For local development/testing, this shouldn't be needed. if os.environ.get("SHARD", None): shard_str = "shard_{}".format(os.environ.get("SHARD")) + BOK_CHOY_REPORT_DIR = BOK_CHOY_REPORT_DIR / shard_str + BOK_CHOY_LOG_DIR = BOK_CHOY_LOG_DIR / shard_str + + # The stubs package is currently located in the Django app called "terrain" + # from when they were used by both the bok-choy and lettuce (deprecated) acceptance tests + BOK_CHOY_STUB_DIR = REPO_ROOT / "common" / "djangoapps" / "terrain" # Directory that videos are served from VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video" - PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" + PRINT_SETTINGS_LOG_FILE = BOK_CHOY_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one - FRONTEND_TEST_SERVER_HOST = os.environ.get('FRONTEND_TEST_SERVER_HOSTNAME', '0.0.0.0') - USING_DOCKER = FRONTEND_TEST_SERVER_HOST != '0.0.0.0' + SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0') + USING_DOCKER = SERVER_HOST != '0.0.0.0' + SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy' DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' TEST_SETTINGS = 'test' + BOK_CHOY_SERVERS = { + 'lms': { + 'host': SERVER_HOST, + 'port': os.environ.get('BOK_CHOY_LMS_PORT', '8003'), + 'log': BOK_CHOY_LOG_DIR / "bok_choy_lms.log" + }, + 'cms': { + 'host': SERVER_HOST, + 'port': os.environ.get('BOK_CHOY_CMS_PORT', '8031'), + 'log': BOK_CHOY_LOG_DIR / "bok_choy_studio.log" + } + } + + BOK_CHOY_STUBS = { + + 'xqueue': { + 'port': 8040, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_xqueue.log", + 'config': 'register_submission_url=http://0.0.0.0:8041/test/register_submission', + }, + + 'ora': { + 'port': 8041, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_ora.log", + 'config': '', + }, + + 'comments': { + 'port': 4567, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_comments.log", + }, + + 'video': { + 'port': 8777, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_video_sources.log", + 'config': f"root_dir={VIDEO_SOURCE_DIR}", + }, + + 'youtube': { + 'port': 9080, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_youtube.log", + }, + + 'edxnotes': { + 'port': 8042, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_edxnotes.log", + }, + + 'ecommerce': { + 'port': 8043, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_ecommerce.log", + }, + + 'catalog': { + 'port': 8091, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_catalog.log", + }, + + 'lti': { + 'port': 8765, + 'log': BOK_CHOY_LOG_DIR / "bok_choy_lti.log", + }, + } + # Mongo databases that will be dropped before/after the tests run - MONGO_HOST = 'localhost' + MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost' + BOK_CHOY_MONGO_DATABASE = "test" + BOK_CHOY_CACHE_HOST = 'edx.devstack.memcached' if USING_DOCKER else '0.0.0.0' + BOK_CHOY_CACHE = memcache.Client([f'{BOK_CHOY_CACHE_HOST}:11211'], debug=0) # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 0f3b0105cd51..bc583442dcdf 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -142,6 +142,8 @@ bleach[css]==6.1.0 # ora2 # xblock-drag-and-drop-v2 # xblock-poll +bok-choy==2.0.2 + # via -r requirements/edx/testing.txt boto==2.39.0 # via # -c requirements/edx/../constraints.txt @@ -1116,6 +1118,7 @@ lazy==1.6 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # acid-xblock + # bok-choy # lti-consumer-xblock # ora2 # xblock @@ -1803,6 +1806,10 @@ scipy==1.7.3 # -r requirements/edx/testing.txt # chem # openedx-calc +selenium==3.141.0 + # via + # -r requirements/edx/testing.txt + # bok-choy semantic-version==2.10.0 # via # -r requirements/edx/doc.txt @@ -2113,6 +2120,7 @@ urllib3==1.26.17 # pact-python # py2neo # requests + # selenium # snowflake-connector-python user-util==1.0.0 # via diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index b7806b027e73..93310adc88f1 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -18,6 +18,7 @@ -r coverage.txt # Utilities for calculating test coverage beautifulsoup4 # Library for extracting data from HTML and XML files +bok-choy # Framework for browser automation tests, based on selenium code-annotations # Perform code annotation checking, such as for PII annotations cssselect # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery ddt # Run a test case multiple times with different input; used in many, many of our tests @@ -40,6 +41,7 @@ pytest-json-report # Output json formatted warnings after running pytest pytest-metadata==1.8.0 # To prevent 'make upgrade' failure, dependency of pytest-json-report pytest-randomly # pytest plugin to randomly order tests pytest-xdist[psutil] # Parallel execution of tests on multiple CPU cores or hosts +selenium # Browser automation library, used for acceptance tests singledispatch # Backport of functools.singledispatch from Python 3.4+, used in tests of XBlock rendering testfixtures # Provides a LogCapture utility used by several tests tox # virtualenv management for tests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 449aa216be6d..218485e8faf5 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -106,6 +106,8 @@ bleach[css]==6.1.0 # ora2 # xblock-drag-and-drop-v2 # xblock-poll +bok-choy==2.0.2 + # via -r requirements/edx/testing.in boto==2.39.0 # via # -c requirements/edx/../constraints.txt @@ -847,6 +849,7 @@ lazy==1.6 # via # -r requirements/edx/base.txt # acid-xblock + # bok-choy # lti-consumer-xblock # ora2 # xblock @@ -1364,6 +1367,10 @@ scipy==1.7.3 # -r requirements/edx/base.txt # chem # openedx-calc +selenium==3.141.0 + # via + # -r requirements/edx/testing.in + # bok-choy semantic-version==2.10.0 # via # -r requirements/edx/base.txt @@ -1557,6 +1564,7 @@ urllib3==1.26.17 # pact-python # py2neo # requests + # selenium # snowflake-connector-python user-util==1.0.0 # via -r requirements/edx/base.txt diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh index f854a3e36cda..5aabf8fde4c2 100755 --- a/scripts/generic-ci-tests.sh +++ b/scripts/generic-ci-tests.sh @@ -20,10 +20,11 @@ set -e # - "pavelib-unit": Run Python unit tests from the pavelib/lib directory # - "pavelib-js-unit": Run the JavaScript tests and the Python unit # tests from the pavelib/lib directory +# - "bok-choy": Run acceptance tests that use the bok-choy framework # # `SHARD` is a number indicating which subset of the tests to build. # -# For "lms-unit", the tests are put into shard groups +# For "bok-choy" and "lms-unit", the tests are put into shard groups # using the 'attr' decorator (e.g. "@attr(shard=1)"). Anything with # the 'shard=n' attribute will run in the nth shard. If there isn't a # shard explicitly assigned, the test will run in the last shard. diff --git a/scripts/reset-test-db.sh b/scripts/reset-test-db.sh new file mode 100755 index 000000000000..fd751b75dd30 --- /dev/null +++ b/scripts/reset-test-db.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +############################################################################ +# +# reset-test-db.sh +# +# Resets the MySQL test database for the bok-choy acceptance tests. +# +# If it finds a cached schema and migration history, it will start +# from the cached version to speed up migrations. +# +# If no cached database exists, it will create one. This can be +# checked into the repo to speed up future tests. +# +# Note that we do NOT want to re-use the cache between test runs! +# A newer commit could introduce migrations that do not exist +# in other commits, which could cause migrations to fail in the other +# commits. +# +# For this reason, we always use a cache that was committed to master +# at the time the branch was created. +# +############################################################################ + +# Fail fast +set -e + +DB_CACHE_DIR="common/test/db_cache" + +if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then + MYSQL_HOST="" + SETTINGS="bok_choy" +else + MYSQL_HOST="--host=edx.devstack.mysql80" + SETTINGS="bok_choy_docker" +fi + +for i in "$@"; do + case $i in + -r|--rebuild_cache) + REBUILD_CACHE=true + ;; + -m|--migrations) + APPLY_MIGRATIONS=true + ;; + -c|--calculate_migrations) + CALCULATE_MIGRATIONS=true + ;; + -u|--use-existing-db) + USE_EXISTING_DB=true + ;; + esac +done + +declare -A databases +declare -a database_order +databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test") +database_order=("default" "student_module_history") + +calculate_migrations() { + echo "Calculating migrations for fingerprinting." + output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml" + # Redirect stdout to /dev/null because the script will print + # out all migrations to both stdout and the output file. + ./manage.py lms --settings "$SETTINGS" show_unapplied_migrations --database "$db" --output_file "$output_file" 1>/dev/null +} + +run_migrations() { + echo "Running the lms migrations on the $db bok_choy DB." + ./manage.py lms --settings "$SETTINGS" migrate --database "$db" --traceback --noinput + echo "Running the cms migrations on the $db bok_choy DB." + ./manage.py cms --settings "$SETTINGS" migrate --database "$db" --traceback --noinput +} + +load_cache_into_db() { + echo "Loading the schema from the filesystem into the $db MySQL DB." + mysql "$MYSQL_HOST" -u root "${databases["$db"]}" < "$DB_CACHE_DIR/bok_choy_schema_$db.sql" + echo "Loading the fixture data from the filesystem into the $db MySQL DB." + ./manage.py lms --settings "$SETTINGS" loaddata --database "$db" "$DB_CACHE_DIR/bok_choy_data_$db.json" + echo "Loading the migration data from the filesystem into the $db MySQL DB." + mysql "$MYSQL_HOST" -u root "${databases["$db"]}" < "$DB_CACHE_DIR/bok_choy_migrations_data_$db.sql" +} + +rebuild_cache_for_db() { + # Make sure the DB has all migrations applied + run_migrations + + # Dump the schema and data to the cache + echo "Using the dumpdata command to save the $db fixture data to the filesystem." + ./manage.py lms --settings "$SETTINGS" dumpdata --database "$db" > "$DB_CACHE_DIR/bok_choy_data_$db.json" --exclude=api_admin.Catalog + echo "Saving the schema of the $db bok_choy DB to the filesystem." + mysqldump "$MYSQL_HOST" -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > "$DB_CACHE_DIR/bok_choy_schema_$db.sql" + + # dump_data does not dump the django_migrations table so we do it separately. + echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem." + mysqldump $MYSQL_HOST -u root --no-create-info --skip-comments --skip-dump-date "${databases["$db"]}" django_migrations > "$DB_CACHE_DIR/bok_choy_migrations_data_$db.sql" +} + +for db in "${database_order[@]}"; do + if ! [[ $USE_EXISTING_DB ]]; then + echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root + + # Clear out the test database + # + # We are using the reset_db command which uses "DROP DATABASE" and + # "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack + # or a jenkins worker environment) that already ran tests on another commit that had + # different migrations that created, dropped, or altered tables. + echo "Issuing a reset_db command to the $db bok_choy MySQL database." + ./manage.py lms --settings "$SETTINGS" reset_db --traceback --router "$db" + fi + + if ! [[ $CALCULATE_MIGRATIONS ]]; then + # If there are cached database schemas/data, then load them. + # If they are missing, then we will want to build new cache files even if + # not explicitly directed to do so via arguments passed to this script. + if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then + REBUILD_CACHE=true + else + load_cache_into_db + fi + fi +done + +if [[ $REBUILD_CACHE ]]; then + echo "Cleaning the DB cache directory and building new files." + mkdir -p $DB_CACHE_DIR && rm -f $DB_CACHE_DIR/bok_choy* + + for db in "${database_order[@]}"; do + rebuild_cache_for_db + done +elif [[ $APPLY_MIGRATIONS ]]; then + for db in "${database_order[@]}"; do + run_migrations + done +elif [[ $CALCULATE_MIGRATIONS ]]; then + for db in "${database_order[@]}"; do + calculate_migrations + done +fi diff --git a/tox.ini b/tox.ini index 80eb6b50977c..689407650699 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,10 @@ setenv = PYTHONHASHSEED=0 TOXENV={envname} passenv = - FRONTEND_TEST_SERVER_CMS_PORT - FRONTEND_TEST_SERVER_HOSTNAME - FRONTEND_TEST_SERVER_LMS_PORT + BOK_CHOY_CMS_PORT + BOKCHOY_HEADLESS + BOK_CHOY_HOSTNAME + BOK_CHOY_LMS_PORT DISABLE_COURSEENROLLMENT_HISTORY DISPLAY DJANGO_SETTINGS_MODULE diff --git a/xmodule/modulestore/tests/mongo_connection.py b/xmodule/modulestore/tests/mongo_connection.py index a0ffe7e7c6f1..8da1fbd42387 100644 --- a/xmodule/modulestore/tests/mongo_connection.py +++ b/xmodule/modulestore/tests/mongo_connection.py @@ -9,4 +9,5 @@ import os MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) -MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') +MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', + 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost') diff --git a/xmodule/tests/test_mongo_utils.py b/xmodule/tests/test_mongo_utils.py index cfa08ba15708..94182b24a725 100644 --- a/xmodule/tests/test_mongo_utils.py +++ b/xmodule/tests/test_mongo_utils.py @@ -3,6 +3,7 @@ """ +import os from unittest import TestCase from uuid import uuid4 @@ -28,7 +29,7 @@ def test_connect_to_mongo_read_preference(self, enum_name, mongos_name, expected """ Test that read_preference parameter gets converted to a valid pymongo read preference. """ - host = 'localhost' + host = 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost' db = 'test_read_preference_%s' % uuid4().hex # Support for read_preference given in constant name form (ie. PRIMARY, SECONDARY_PREFERRED) connection = connect_to_mongodb(db, host, read_preference=enum_name) diff --git a/xmodule/tests/test_video.py b/xmodule/tests/test_video.py index e5180843d7c6..7daa6bdfe097 100644 --- a/xmodule/tests/test_video.py +++ b/xmodule/tests/test_video.py @@ -15,6 +15,7 @@ import datetime import json +import os import shutil import unittest from tempfile import mkdtemp @@ -936,7 +937,7 @@ def test_student_view_data_with_hls_flag(self, mock_get_video_info, mock_get_vid @patch.object(settings, 'CONTENTSTORE', create=True, new={ 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'DOC_STORE_CONFIG': { - 'host': 'localhost', + 'host': 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost', 'db': 'test_xcontent_%s' % uuid4().hex, }, # allow for additional options that can be keyed on a name, e.g. 'trashcan'