From 6eb67860aa925ccdfe0a06cd90a62fbec84f5c14 Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 09:54:14 -0500 Subject: [PATCH 1/8] feat: adding custom repos for cloning --- repo.sh | 144 ++++++++++++++++++++++++++------------------------------ 1 file changed, 68 insertions(+), 76 deletions(-) diff --git a/repo.sh b/repo.sh index ac7ec56779..c9f4454eb3 100755 --- a/repo.sh +++ b/repo.sh @@ -18,85 +18,75 @@ else fi # When you add new services should add them to both repos and ssh_repos -# (or non_release_repos and non_release_ssh_repos if they are not part -# of Open edX releases). -repos=( - "https://github.com/openedx/course-discovery.git" - "https://github.com/openedx/credentials.git" - "https://github.com/openedx/cs_comments_service.git" - "https://github.com/openedx/ecommerce.git" - "https://github.com/edx/edx-e2e-tests.git" - "https://github.com/openedx/edx-notes-api.git" - "https://github.com/openedx/edx-platform.git" - "https://github.com/openedx/xqueue.git" - "https://github.com/openedx/edx-analytics-dashboard.git" - "https://github.com/openedx/frontend-app-gradebook.git" - "https://github.com/openedx/frontend-app-payment.git" - "https://github.com/openedx/frontend-app-publisher.git" - "https://github.com/openedx/edx-analytics-dashboard.git" - "https://github.com/openedx/edx-analytics-data-api.git" +# of Open edX releases. +declare -A repos=( + ["https://github.com/Pearson-Advance/course-discovery.git"]="pearson-release/olive.main" + ["https://github.com/openedx/credentials.git"]="open-release/olive.master" + ["https://github.com/openedx/cs_comments_service.git"]="open-release/olive.master" + ["https://github.com/Pearson-Advance/ecommerce.git"]="pearson-release/olive.main" + ["https://github.com/edx/edx-e2e-tests.git"]="master" + ["https://github.com/Pearson-Advance/edx-platform.git"]="pearson-release/olive.main" + ["https://github.com/openedx/xqueue.git"]="open-release/olive.master" + ["https://github.com/openedx/edx-analytics-dashboard.git"]="open-release/olive.master" + ["https://github.com/openedx/frontend-app-gradebook.git"]="open-release/olive.master" + ["https://github.com/openedx/frontend-app-payment.git"]="open-release/olive.master" + ["https://github.com/openedx/frontend-app-publisher.git"]="open-release/olive.master" + ["https://github.com/openedx/edx-analytics-dashboard.git"]="open-release/olive.master" + ["https://github.com/openedx/edx-analytics-data-api.git"]="open-release/olive.master" + ["https://github.com/openedx/frontend-app-authn.git"]="open-release/olive.master" + ["https://github.com/openedx/frontend-app-course-authoring.git"]="open-release/olive.master" + ["https://github.com/Pearson-Advance/frontend-app-learning.git"]="pearson-release/olive.main" + ["https://github.com/openedx/frontend-app-library-authoring.git"]="master" + ["https://github.com/openedx/registrar.git"]="master" + ["https://github.com/openedx/frontend-app-program-console.git"]="master" + ["https://github.com/openedx/frontend-app-account.git"]="open-release/olive.master" + ["https://github.com/openedx/frontend-app-profile.git"]="open-release/olive.master" + ["https://github.com/edx/frontend-app-ora-grading.git"]="open-release/olive.master" + ["https://github.com/Pearson-Advance/openedx-themes.git"]="pearson-release/olive.main" ) -non_release_repos=( - "https://github.com/openedx/frontend-app-authn.git" - "https://github.com/openedx/frontend-app-course-authoring.git" - "https://github.com/openedx/frontend-app-learning.git" - "https://github.com/openedx/frontend-app-library-authoring.git" - "https://github.com/openedx/registrar.git" - "https://github.com/openedx/frontend-app-program-console.git" - "https://github.com/openedx/frontend-app-account.git" - "https://github.com/openedx/frontend-app-profile.git" - "https://github.com/edx/frontend-app-ora-grading.git" +declare -A ssh_repos=( + ["git@github.com:Pearson-Advance/course-discovery.git"]="pearson-release/olive.main" + ["git@github.com:openedx/credentials.git"]="open-release/olive.master" + ["git@github.com:openedx/cs_comments_service.git"]="open-release/olive.master" + ["git@github.com:Pearson-Advance/ecommerce.git"]="pearson-release/olive.main" + ["git@github.com:edx/edx-e2e-tests.git"]="master" + ["git@github.com:Pearson-Advance/edx-platform.git"]="pearson-release/olive.main" + ["git@github.com:openedx/xqueue.git"]="open-release/olive.master" + ["git@github.com:openedx/edx-analytics-dashboard.git"]="open-release/olive.master" + ["git@github.com:openedx/frontend-app-gradebook.git"]="open-release/olive.master" + ["git@github.com:openedx/frontend-app-payment.git"]="open-release/olive.master" + ["git@github.com:openedx/frontend-app-publisher.git"]="open-release/olive.master" + ["git@github.com:openedx/edx-analytics-dashboard.git"]="open-release/olive.master" + ["git@github.com:openedx/edx-analytics-data-api.git"]="open-release/olive.master" + ["git@github.com:openedx/frontend-app-authn.git"]="open-release/olive.master" + ["git@github.com:openedx/frontend-app-course-authoring.git"]="open-release/olive.master" + ["git@github.com:Pearson-Advance/frontend-app-learning.git"]="pearson-release/olive.main" + ["git@github.com:openedx/frontend-app-library-authoring.git"]="master" + ["git@github.com:openedx/registrar.git"]="master" + ["git@github.com:openedx/frontend-app-program-console.git"]="master" + ["git@github.com:openedx/frontend-app-account.git"]="open-release/olive.master" + ["git@github.com:openedx/frontend-app-profile.git"]="open-release/olive.master" + ["git@github.com:edx/frontend-app-ora-grading.git"]="open-release/olive.master" + ["git@github.com:Pearson-Advance/openedx-themes.git"]="pearson-release/olive.main" ) -ssh_repos=( - "git@github.com:openedx/course-discovery.git" - "git@github.com:openedx/credentials.git" - "git@github.com:openedx/cs_comments_service.git" - "git@github.com:openedx/ecommerce.git" - "git@github.com:edx/edx-e2e-tests.git" - "git@github.com:openedx/edx-notes-api.git" - "git@github.com:openedx/edx-platform.git" - "git@github.com:openedx/xqueue.git" - "git@github.com:openedx/edx-analytics-dashboard.git" - "git@github.com:openedx/frontend-app-gradebook.git" - "git@github.com:openedx/frontend-app-payment.git" - "git@github.com:openedx/frontend-app-publisher.git" - "git@github.com:openedx/edx-analytics-dashboard.git" - "git@github.com:openedx/edx-analytics-data-api.git" ) -non_release_ssh_repos=( - "git@github.com:openedx/frontend-app-authn.git" - "git@github.com:openedx/frontend-app-course-authoring.git" - "git@github.com:openedx/frontend-app-learning.git" - "git@github.com:openedx/frontend-app-library-authoring.git" - "git@github.com:openedx/registrar.git" - "git@github.com:openedx/frontend-app-program-console.git" - "git@github.com:openedx/frontend-app-account.git" - "git@github.com:openedx/frontend-app-profile.git" - "git@github.com:edx/frontend-app-ora-grading.git" ) -private_repos=( - # Needed to run whitelabel tests. - "https://github.com/edx/edx-themes.git" ) -if [ -n "${OPENEDX_RELEASE}" ]; then - OPENEDX_GIT_BRANCH=open-release/${OPENEDX_RELEASE} -else - repos+=("${non_release_repos[@]}") - ssh_repos+=("${non_release_ssh_repos[@]}") -fi name_pattern=".*/(.*).git" _checkout () { + eval "declare -A repositories="${1#*=} + repos_to_checkout=("$@") - for repo in "${repos_to_checkout[@]}" + for repo in "${!repositories[@]}" do # Use Bash's regex match operator to capture the name of the repo. # Results of the match are saved to an array called $BASH_REMATCH. @@ -106,7 +96,7 @@ _checkout () # If a directory exists and it is nonempty, assume the repo has been cloned. if [ -d "$name" ] && [ -n "$(ls -A "$name" 2>/dev/null)" ]; then cd "$name" - _checkout_and_update_branch + _checkout_and_update_branch ${repositories[$repo]} cd .. fi done @@ -114,14 +104,14 @@ _checkout () checkout () { - _checkout "${repos[@]}" + _checkout "$(declare -p repos)" } _clone () { + eval "declare -A repositories="${1#*=} - repos_to_clone=("$@") - for repo in "${repos_to_clone[@]}" + for repo in "${!repositories[@]}" do # Use Bash's regex match operator to capture the name of the repo. # Results of the match are saved to an array called $BASH_REMATCH. @@ -137,7 +127,7 @@ _clone () fi printf "The [%s] repo is already checked out. Checking for updates.\n" "$name" cd "${DEVSTACK_WORKSPACE}/${name}" - _checkout_and_update_branch + _checkout_and_update_branch "${repositories[$repo]}" cd .. else if [ -n "${OPENEDX_GIT_BRANCH:-}" ]; then @@ -146,9 +136,9 @@ _clone () CLONE_BRANCH="" fi if [ "${SHALLOW_CLONE}" == "1" ]; then - git clone ${CLONE_BRANCH} -c core.symlinks=true --depth=1 "${repo}" + git clone --single-branch -b ${repositories[$repo]} -c core.symlinks=true --depth=1 "${repo}" else - git clone ${CLONE_BRANCH} -c core.symlinks=true "${repo}" + git clone --single-branch -b ${repositories[$repo]} -c core.symlinks=true "${repo}" fi fi done @@ -165,28 +155,30 @@ _checkout_and_update_branch () CHECKOUT_BRANCH=$(git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@') fi echo "Checking out branch ${CHECKOUT_BRANCH}" - if [ "${BRANCH_NAME}" == "${CHECKOUT_BRANCH}" ]; then - git pull origin ${CHECKOUT_BRANCH} + DESIRED_BRANCH=$1 + + if [ "${BRANCH_NAME}" == "${DESIRED_BRANCH}" ]; then + git pull origin ${DESIRED_BRANCH} else - git fetch origin ${CHECKOUT_BRANCH}:${CHECKOUT_BRANCH} - git checkout ${CHECKOUT_BRANCH} + git fetch origin ${DESIRED_BRANCH}:${DESIRED_BRANCH} + git checkout ${DESIRED_BRANCH} fi find . -name '*.pyc' -not -path './.git/*' -delete } clone () { - _clone "${repos[@]}" + _clone "$(declare -p repos)" } clone_ssh () { - _clone "${ssh_repos[@]}" + _clone "$(declare -p ssh_repos)" } clone_private () { - _clone "${private_repos[@]}" + _clone "$(declare -p private_repos)" } reset () From c7433389564224a087f1678cae414152396ea0c9 Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 09:54:35 -0500 Subject: [PATCH 2/8] feat: adding private Pearson repos cloning --- docker-compose-host.yml | 6 ++++++ docker-compose.yml | 20 ++++++++++++++++++++ repo.sh | 19 +++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/docker-compose-host.yml b/docker-compose-host.yml index 82d8d40cac..f3f54b2641 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -118,6 +118,11 @@ services: - ${DEVSTACK_WORKSPACE}/frontend-app-publisher:/edx/app/frontend-app-publisher - frontend_app_publisher_node_modules:/edx/app/frontend-app-publisher/node_modules - ${DEVSTACK_WORKSPACE}/src:/edx/app/src + frontend-app-sidebar-navigation: + volumes: + - ${DEVSTACK_WORKSPACE}/frontend-app-sidebar-navigation:/edx/app/frontend-app-sidebar-navigation + - frontend_app_sidebar_navigation_node_modules:/edx/app/frontend-app-sidebar-navigation/node_modules + - ${DEVSTACK_WORKSPACE}/src:/edx/app/src volumes: credentials_node_modules: @@ -138,6 +143,7 @@ volumes: frontend_app_payment_node_modules: frontend_app_program_console_node_modules: frontend_app_publisher_node_modules: + frontend_app_sidebar_navigation_node_modules: credentials_tox: discovery_tox: ecommerce_tox: diff --git a/docker-compose.yml b/docker-compose.yml index d4ec1ecd11..7373cfece1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -826,6 +826,26 @@ services: depends_on: - lms + # ========================================================================== + # Pearson Microfrontends + # ========================================================================== + + frontend-app-sidebar-navigation: + extends: + file: microfrontend.yml + service: microfrontend + working_dir: /edx/app/frontend-app-sidebar-navigation + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-sidebar-navigation" + networks: + default: + aliases: + - edx.devstack.frontend-app-sidebar-navigation + ports: + - "9090:9090" + depends_on: + - lms + - frontend-app-learning + volumes: coursegraph_data: discovery_assets: diff --git a/repo.sh b/repo.sh index c9f4454eb3..0e604c6086 100755 --- a/repo.sh +++ b/repo.sh @@ -71,15 +71,34 @@ declare -A ssh_repos=( ["git@github.com:Pearson-Advance/openedx-themes.git"]="pearson-release/olive.main" ) +declare -A private_repos=( + # Needed to run whitelabel tests. + ["https://github.com/edx/edx-themes.git"]="master" ) +declare -A pearson_private_repos=( + ["https://github.com/Pearson-Advance/frontend-app-sidebar-navigation.git"]="pearson-release/olive.main" ) +declare -A pearson_private_ssh_repos=( + ["git@github.com:Pearson-Advance/frontend-app-sidebar-navigation.git"]="pearson-release/olive.main" ) name_pattern=".*/(.*).git" +if git config --global user.email &> /dev/null && git config --global user.name &> /dev/null +then + for key in "${!pearson_private_repos[@]}" + do + repos["$key"]="${pearson_private_repos[$key]}" + done + for key in "${!pearson_private_ssh_repos[@]}" + do + ssh_repos["$key"]="${pearson_private_ssh_repos[$key]}" + done +fi + _checkout () { eval "declare -A repositories="${1#*=} From 235d358072cb5f751deb3b0ae7aa732e9279df55 Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 10:58:47 -0500 Subject: [PATCH 3/8] feat: changing images to olive.master or to paops images --- docker-compose.yml | 14 +++++++------- options.mk | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7373cfece1..1812376218 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -319,7 +319,7 @@ services: SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2 ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 - image: edxops/credentials:${OPENEDX_RELEASE:-latest} + image: edxops/credentials:latest networks: default: aliases: @@ -345,7 +345,7 @@ services: TEST_ELASTICSEARCH_URL: "edx.devstack.elasticsearch710" ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 - image: edxops/discovery:${OPENEDX_RELEASE:-latest} + image: paops/discovery-devstack:${OPENEDX_RELEASE:-latest} networks: default: aliases: @@ -370,7 +370,7 @@ services: environment: DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 - image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + image: paops/ecommerce-devstack:olive.master networks: default: aliases: @@ -449,7 +449,7 @@ services: EDXAPP_TEST_MONGO_HOST: edx.devstack.mongo NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 - image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + image: paops/edxapp:${OPENEDX_RELEASE:-latest} networks: default: aliases: @@ -543,7 +543,7 @@ services: CELERY_BROKER_VHOST: 10 CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 - image: edxops/registrar:${OPENEDX_RELEASE:-latest} + image: edxops/registrar:latest networks: default: aliases: @@ -577,7 +577,7 @@ services: CELERY_BROKER_VHOST: 10 CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 - image: edxops/registrar:${OPENEDX_RELEASE:-latest} + image: edxops/registrar:latest networks: default: aliases: @@ -609,7 +609,7 @@ services: EDXAPP_TEST_MONGO_HOST: edx.devstack.mongo NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 - image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + image: paops/edxapp:${OPENEDX_RELEASE:-latest} networks: default: aliases: diff --git a/options.mk b/options.mk index dc2b453001..1b994f39d6 100644 --- a/options.mk +++ b/options.mk @@ -23,7 +23,7 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # If it is defined in options.local.mk or the environment, then Devstack will try # to use the Docker images and Git repo branches that correspond to that release. # If the release does not exist, you will see errors. -# OPENEDX_RELEASE ?= +OPENEDX_RELEASE ?= olive.master # Name of Docker Compose project. # Volumes and network are namespaced based on this value, @@ -32,6 +32,7 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # If OPENEDX_RELAESE is defined, defaults to `devstack-${OPENEDX_RELEASE}`; # otherwise, it defaults to `devstack`. # Be sure to bring down services before changing the value of `COMPOSE_PROJECT_NAME`. + ifdef OPENEDX_RELEASE COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} else From 963b808f1232eec0f064fa2b7f465d0cd8f78989 Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 11:02:41 -0500 Subject: [PATCH 4/8] feat: changing volumes directory so they are created in the devstack folder --- docker-compose-host.yml | 13 +++++-------- docker-compose.yml | 19 +++++++------------ 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/docker-compose-host.yml b/docker-compose-host.yml index f3f54b2641..5dcebc4540 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -5,19 +5,19 @@ services: credentials: volumes: - ${DEVSTACK_WORKSPACE}/credentials:/edx/app/credentials/credentials - - credentials_node_modules:/edx/app/credentials/credentials/node_modules + - ../volumes/credentials_node_modules:/edx/app/credentials/credentials/node_modules - credentials_tox:/edx/app/credentials/credentials/.tox - ${DEVSTACK_WORKSPACE}/src:/edx/src discovery: volumes: - ${DEVSTACK_WORKSPACE}/course-discovery:/edx/app/discovery/discovery - - discovery_node_modules:/edx/app/discovery/discovery/node_modules + - ../volumes/discovery_node_modules:/edx/app/discovery/discovery/node_modules - discovery_tox:/edx/app/discovery/discovery/.tox - ${DEVSTACK_WORKSPACE}/src:/edx/src ecommerce: volumes: - ${DEVSTACK_WORKSPACE}/ecommerce:/edx/app/ecommerce/ecommerce - - ecommerce_node_modules:/edx/app/ecommerce/ecommerce/node_modules + - ../volumes/ecommerce_node_modules:/edx/app/ecommerce/ecommerce/node_modules - ecommerce_tox:/edx/app/ecommerce/ecommerce/.tox - ${DEVSTACK_WORKSPACE}/src:/edx/src forum: @@ -27,7 +27,7 @@ services: volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform - edxapp_media:/edx/var/edxapp/media - - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules + - ../volumes/edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - edxapp_tox:/edx/app/edxapp/edx-platform/.tox - edxapp_uploads:/edx/var/edxapp/uploads - ${DEVSTACK_WORKSPACE}/src:/edx/src @@ -45,7 +45,7 @@ services: volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform - edxapp_media:/edx/var/edxapp/media - - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules + - ../volumes/edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - edxapp_tox:/edx/app/edxapp/edx-platform/.tox - edxapp_uploads:/edx/var/edxapp/uploads - ${DEVSTACK_WORKSPACE}/src:/edx/src @@ -126,11 +126,8 @@ services: volumes: credentials_node_modules: - discovery_node_modules: - ecommerce_node_modules: insights_node_modules: edxapp_media: - edxapp_node_modules: edxapp_uploads: frontend_app_account_node_modules: frontend_app_profile_node_modules: diff --git a/docker-compose.yml b/docker-compose.yml index 1812376218..fd5e9e07e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,8 +77,8 @@ services: # - "9200:9200" # - "9300:9300" volumes: - - elasticsearch_data:/usr/share/elasticsearch/data - - elasticsearch_data:/usr/share/elasticsearch/logs + - ../volumes/elasticsearch_data:/usr/share/elasticsearch/data + - ../volumes/elasticsearch_data:/usr/share/elasticsearch/logs # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. elasticsearch7: @@ -235,7 +235,7 @@ services: ports: - "27017:27017" volumes: - - mongo_data:/data/db + - ../volumes/mongo_data:/data/db mysql57: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci @@ -252,7 +252,7 @@ services: ports: - "3506:3306" volumes: - - mysql57_data:/var/lib/mysql + - ../volumes/mysql57_data:/var/lib/mysql redis: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis" @@ -353,7 +353,7 @@ services: ports: - "18381:18381" volumes: - - discovery_assets:/edx/var/discovery/ + - ../volumes/discovery_assets:/edx/var/discovery/ ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' @@ -513,6 +513,7 @@ services: - "19001:19001" volumes: - /edx/var/analyticsapi + - ../volumes/edxapp_lms_assets:/edx/var/edxapp/staticfiles/ registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' @@ -621,7 +622,7 @@ services: # - "18103:18103" # - "18131:18131" volumes: - - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ + - ../volumes/edxapp_studio_assets:/edx/var/edxapp/staticfiles/ xqueue: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" @@ -848,13 +849,7 @@ services: volumes: coursegraph_data: - discovery_assets: devpi_data: - edxapp_lms_assets: - edxapp_studio_assets: - elasticsearch_data: elasticsearch7_data: elasticsearch710_data: - mongo_data: opensearch12_data: - mysql57_data: From f76ead939714db428bf1c0f091f8000ae484513d Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 12:22:23 -0500 Subject: [PATCH 5/8] feat: adding openedx-themes --- docker-compose-themes.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-themes.yml b/docker-compose-themes.yml index 94de501d94..b900b412e2 100644 --- a/docker-compose-themes.yml +++ b/docker-compose-themes.yml @@ -3,13 +3,13 @@ version: "2.1" services: discovery: volumes: - - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes + - ${DEVSTACK_WORKSPACE}/openedx-themes:/edx/app/edx-themes ecommerce: volumes: - - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes + - ${DEVSTACK_WORKSPACE}/openedx-themes:/edx/app/edx-themes lms: volumes: - - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes + - ${DEVSTACK_WORKSPACE}/openedx-themes:/edx/app/edx-themes studio: volumes: - - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes + - ${DEVSTACK_WORKSPACE}/openedx-themes:/edx/app/edx-themes From 52b47eb3f83467e7a3a360f52ac0e26fa2866d51 Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 12:24:52 -0500 Subject: [PATCH 6/8] feat: optimizing devstack by removing services --- .github/workflows/provisioning-tests.yml | 2 +- docker-compose-host.yml | 24 +- docker-compose.yml | 271 ++++++++++++----------- options.mk | 11 +- provision.sh | 13 +- 5 files changed, 165 insertions(+), 156 deletions(-) diff --git a/.github/workflows/provisioning-tests.yml b/.github/workflows/provisioning-tests.yml index c07f44be18..6f93397244 100644 --- a/.github/workflows/provisioning-tests.yml +++ b/.github/workflows/provisioning-tests.yml @@ -26,7 +26,7 @@ jobs: os: - ubuntu-20.04 # Ubuntu 20.04 "Focal Fossa" python-version: [ '3.8' ] - services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue] + services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, credentials+lms, xqueue] fail-fast: false # some services can be flaky; let others run to completion even if one fails steps: diff --git a/docker-compose-host.yml b/docker-compose-host.yml index 5dcebc4540..20ba001bed 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -31,10 +31,10 @@ services: - edxapp_tox:/edx/app/edxapp/edx-platform/.tox - edxapp_uploads:/edx/var/edxapp/uploads - ${DEVSTACK_WORKSPACE}/src:/edx/src - edx_notes_api: - volumes: - - ${DEVSTACK_WORKSPACE}/edx-notes-api:/edx/app/edx_notes_api/edx_notes_api - - ${DEVSTACK_WORKSPACE}/src:/edx/src + # edx_notes_api: + # volumes: + # - ${DEVSTACK_WORKSPACE}/edx-notes-api:/edx/app/edx_notes_api/edx_notes_api + # - ${DEVSTACK_WORKSPACE}/src:/edx/src registrar: volumes: - ${DEVSTACK_WORKSPACE}/registrar:/edx/app/registrar/registrar @@ -49,14 +49,14 @@ services: - edxapp_tox:/edx/app/edxapp/edx-platform/.tox - edxapp_uploads:/edx/var/edxapp/uploads - ${DEVSTACK_WORKSPACE}/src:/edx/src - insights: - volumes: - - ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights - - insights_node_modules:/edx/app/insights/insights/node_modules - analyticsapi: - volumes: - - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api - - ${DEVSTACK_WORKSPACE}/src:/edx/src + # insights: + # volumes: + # - ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights + # - insights_node_modules:/edx/app/insights/insights/node_modules + # analyticsapi: + # volumes: + # - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api + # - ${DEVSTACK_WORKSPACE}/src:/edx/src # Note that frontends mount `src` to /edx/app/src instead of /edx/src. # See ADR #5 for rationale. diff --git a/docker-compose.yml b/docker-compose.yml index fd5e9e07e0..ac079c9963 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,25 +31,25 @@ services: - ../edx-e2e-tests/upload_files:/edx/app/e2e/edx-e2e-tests/upload_files - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data - coursegraph: - container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.coursegraph" - hostname: coursegraph.devstack.edx - # Try to keep this in sync with the NEO4J_VERSION declared within - # https://github.com/openedx/configuration/blob/master/playbooks/roles/neo4j - image: neo4j:3.5.28 - networks: - default: - aliases: - - edx.devstack.coursegraph - ports: - - "7474:7474" # Expose Web interface at http://localhost:7474. - - "7687:7687" # Expose Bolt interface at bolt://user:password@localhost:7687. - volumes: - - coursegraph_data:/data - stdin_open: true - tty: true - environment: - NEO4J_AUTH: "neo4j/edx" # Initial username/password for Neo4j Web interface. + # coursegraph: + # container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.coursegraph" + # hostname: coursegraph.devstack.edx + # # Try to keep this in sync with the NEO4J_VERSION declared within + # # https://github.com/openedx/configuration/blob/master/playbooks/roles/neo4j + # image: neo4j:3.5.28 + # networks: + # default: + # aliases: + # - edx.devstack.coursegraph + # ports: + # - "7474:7474" # Expose Web interface at http://localhost:7474. + # - "7687:7687" # Expose Bolt interface at bolt://user:password@localhost:7687. + # volumes: + # - coursegraph_data:/data + # stdin_open: true + # tty: true + # environment: + # NEO4J_AUTH: "neo4j/edx" # Initial username/password for Neo4j Web interface. devpi: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" @@ -119,38 +119,38 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # This is meant to be used to test OS upgrades. - opensearch12: - container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.opensearch12" - hostname: opensearch12.devstack.edx - image: opensearchproject/opensearch:1.2.0 - networks: - default: - aliases: - - edx.devstack.opensearch12 - ports: - - "9202:9200" - - "9600:9600" - volumes: - - opensearch12_data:/usr/share/opensearch/data - environment: - - discovery.type=single-node - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - firefox: - container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" - hostname: firefox.devstack.edx - image: edxops/firefox:${OPENEDX_RELEASE:-latest} - shm_size: 2g - networks: - default: - aliases: - - edx.devstack.firefox - ports: - - "25900:5900" - volumes: # for file uploads - - ../edx-e2e-tests/upload_files:/edx/app/e2e/edx-e2e-tests/upload_files - - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data + # opensearch12: + # container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.opensearch12" + # hostname: opensearch12.devstack.edx + # image: opensearchproject/opensearch:1.2.0 + # networks: + # default: + # aliases: + # - edx.devstack.opensearch12 + # ports: + # - "9202:9200" + # - "9600:9600" + # volumes: + # - opensearch12_data:/usr/share/opensearch/data + # environment: + # - discovery.type=single-node + # - bootstrap.memory_lock=true + # - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + + # firefox: + # container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" + # hostname: firefox.devstack.edx + # image: edxops/firefox:${OPENEDX_RELEASE:-latest} + # shm_size: 2g + # networks: + # default: + # aliases: + # - edx.devstack.firefox + # ports: + # - "25900:5900" + # volumes: # for file uploads + # - ../edx-e2e-tests/upload_files:/edx/app/e2e/edx-e2e-tests/upload_files + # - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data # Events broker kafka: @@ -350,6 +350,8 @@ services: default: aliases: - edx.devstack.discovery + - discovery.pearson.localhost + - discovery.main.localhost ports: - "18381:18381" volumes: @@ -374,38 +376,39 @@ services: networks: default: aliases: - - edx.devstack.ecommerce + - ecommerce.pearson.localhost + - ecommerce.main.localhost ports: - "18130:18130" - edx_notes_api: - # Sleep as a part of start up to give elasticsearch enough time to start up. - command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 4; done' - container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edxnotesapi" - hostname: edx_notes_api.devstack.edx - depends_on: - - devpi - - elasticsearch710 - - lms - - mysql57 - image: edxops/notes:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.edxnotesapi - ports: - - "18120:18120" - environment: - DB_ENGINE: "django.db.backends.mysql" - DB_HOST: "edx.devstack.mysql57" - DB_NAME: "notes" - DB_PASSWORD: "password" - DB_PORT: "3306" - DB_USER: "notes001" - DJANGO_WATCHMAN_TIMEOUT: 30 - ENABLE_DJANGO_TOOLBAR: 1 - ELASTICSEARCH_URL: "http://edx.devstack.elasticsearch710:9200" - ELASTICSEARCH_DSL: "http://edx.devstack.elasticsearch710:9200" + # edx_notes_api: + # # Sleep as a part of start up to give elasticsearch enough time to start up. + # command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 4; done' + # container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edxnotesapi" + # hostname: edx_notes_api.devstack.edx + # depends_on: + # - devpi + # - elasticsearch710 + # - lms + # - mysql57 + # image: edxops/notes:${OPENEDX_RELEASE:-latest} + # networks: + # default: + # aliases: + # - edx.devstack.edxnotesapi + # ports: + # - "18120:18120" + # environment: + # DB_ENGINE: "django.db.backends.mysql" + # DB_HOST: "edx.devstack.mysql57" + # DB_NAME: "notes" + # DB_PASSWORD: "password" + # DB_PORT: "3306" + # DB_USER: "notes001" + # DJANGO_WATCHMAN_TIMEOUT: 30 + # ENABLE_DJANGO_TOOLBAR: 1 + # ELASTICSEARCH_URL: "http://edx.devstack.elasticsearch710:9200" + # ELASTICSEARCH_DSL: "http://edx.devstack.elasticsearch710:9200" forum: command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' @@ -415,7 +418,6 @@ services: - memcached - mongo - elasticsearch710 - - opensearch12 image: edxops/forum:${OPENEDX_RELEASE:-latest} stdin_open: true tty: true @@ -438,7 +440,6 @@ services: - memcached - mongo - mysql57 - - opensearch12 # Allows attachment to the LMS service using 'docker attach '. stdin_open: true tty: true @@ -455,66 +456,67 @@ services: aliases: - edx.devstack.lms - lms.devstack.edx + - lms.pearson.localhost + - lms.main.localhost ports: - "18000:18000" - "19876:19876" # JS test debugging # - "18003:18003" # - "18031:18031" volumes: - - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ - - insights: - command: bash -c 'source /edx/app/insights/insights_env && while true; do python /edx/app/insights/insights/manage.py runserver 0.0.0.0:18110 --settings analytics_dashboard.settings.devstack; sleep 2; done' - container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights" - hostname: insights.devstack.edx - depends_on: - - analyticsapi - - mysql57 - - lms - - memcached - # Allows attachment to the insights service using 'docker attach '. - stdin_open: true - tty: true - environment: - DB_HOST: edx.devstack.mysql57 - DB_NAME: dashboard - DB_PORT: 3306 - DB_USER: analytics001 - DB_PASSWORD: password - LMS_HOST: http://localhost:18000 - DJANGO_SETTINGS_MODULE: analytics_dashboard.settings.devstack - image: edxops/insights:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.insights - ports: - - "18110:18110" - volumes: - - /edx/var/insights/ - - analyticsapi: - image: edxops/analytics_api:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analyticsapi - hostname: analyticsapi - depends_on: - - mysql57 - - elasticsearch710 - command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true; do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done' - stdin_open: true - tty: true - environment: - DB_HOST: edx.devstack.mysql57 - DB_PORT: 3306 - DB_USER: analytics001 - DB_PASSWORD: password - ELASTICSEARCH_LEARNERS_HOST: edx.devstack.elasticsearch710 - ports: - - "19001:19001" - volumes: - - /edx/var/analyticsapi - ../volumes/edxapp_lms_assets:/edx/var/edxapp/staticfiles/ + # insights: + # command: bash -c 'source /edx/app/insights/insights_env && while true; do python /edx/app/insights/insights/manage.py runserver 0.0.0.0:18110 --settings analytics_dashboard.settings.devstack; sleep 2; done' + # container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights" + # hostname: insights.devstack.edx + # depends_on: + # - analyticsapi + # - mysql57 + # - lms + # - memcached + # # Allows attachment to the insights service using 'docker attach '. + # stdin_open: true + # tty: true + # environment: + # DB_HOST: edx.devstack.mysql57 + # DB_NAME: dashboard + # DB_PORT: 3306 + # DB_USER: analytics001 + # DB_PASSWORD: password + # LMS_HOST: http://localhost:18000 + # DJANGO_SETTINGS_MODULE: analytics_dashboard.settings.devstack + # image: edxops/insights:${OPENEDX_RELEASE:-latest} + # networks: + # default: + # aliases: + # - edx.devstack.insights + # ports: + # - "18110:18110" + # volumes: + # - /edx/var/insights/ + + # analyticsapi: + # image: edxops/analytics_api:${OPENEDX_RELEASE:-latest} + # container_name: edx.devstack.analyticsapi + # hostname: analyticsapi + # depends_on: + # - mysql57 + # - elasticsearch710 + # command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true; do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done' + # stdin_open: true + # tty: true + # environment: + # DB_HOST: edx.devstack.mysql57 + # DB_PORT: 3306 + # DB_USER: analytics001 + # DB_PASSWORD: password + # ELASTICSEARCH_LEARNERS_HOST: edx.devstack.elasticsearch710 + # ports: + # - "19001:19001" + # volumes: + # - /edx/var/analyticsapi + registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar" @@ -599,7 +601,6 @@ services: - memcached - mongo - mysql57 - - opensearch12 # Allows attachment to the Studio service using 'docker attach '. stdin_open: true tty: true diff --git a/options.mk b/options.mk index 1b994f39d6..05657c0d6f 100644 --- a/options.mk +++ b/options.mk @@ -62,13 +62,14 @@ ALWAYS_CACHE_PROGRAMS ?= false # The current value was chosen such that it would not change the existing # Devstack behavior. DEFAULT_SERVICES ?= \ -credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-authn+frontend-app-gradebook+frontend-app-payment+frontend-app-publisher+frontend-app-learning+lms+studio +credentials+discovery+ecommerce+forum+frontend-app-authn+frontend-app-gradebook+frontend-app-payment+frontend-app-publisher+frontend-app-learning+lms+studio+frontend-app-sidebar-navigation # All edX services, whether or not they are run by default. # Separated by plus signs. # Separated by plus signs. Listed in alphabetical order for clarity. EDX_SERVICES ?= \ -analyticsapi+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer +analyticsapi+credentials+discovery+ecommerce+forum+frontend-app-account+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer+frontend-app-sidebar-navigation\ +# +edx_notes_api+insights # Services with database migrations. # Should be a subset of $(EDX_SERVICES). @@ -86,9 +87,11 @@ credentials+discovery+ecommerce+lms+registrar+studio # Note: This list should contain _all_ services with static asse to compile ts, even if not # configured to run; the list will be filtered later against $(DEFAULT_SERVICES). ASSET_SERVICES ?= \ -credentials+discovery+ecommerce+insights+lms+registrar+studio +credentials+discovery+ecommerce+lms+registrar+studio\ +# +insights # All third-party services. # Separated by plus signs. Listed in alphabetical order for clarity. THIRD_PARTY_SERVICES ?= \ -chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql57+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica +coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+memcached+mongo+mysql57+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica\ +# +chrome+firefox diff --git a/provision.sh b/provision.sh index 86932ca331..300242c0c9 100755 --- a/provision.sh +++ b/provision.sh @@ -44,14 +44,19 @@ discovery \ credentials \ e2e \ forum \ -notes \ registrar \ xqueue \ -coursegraph \ -insights \ -analyticsapi \ " +# Optional services opted out to decrease the execution time of the provision +# To include these services remove the comenting here an whenever else in the devstack files they are deactivated +# ALL_SERVICES_IN_ORDER+=" \ +# notes \ +# insights \ +# coursegraph \ +# analyticsapi \ +# " + # What should we provision? if [[ $# -eq 0 ]]; then requested_services=$ALL_SERVICES_IN_ORDER From 5bcc790c7e29de6546aeefbf7ed14217c2fb5a43 Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 12:27:54 -0500 Subject: [PATCH 7/8] feat: adding scripts to create new custom sites and its configuration --- edxapp.sql | 2 +- provision-discovery.sh | 6 +++++- provision-ecommerce.sh | 4 +++- provision-lms.sh | 16 +++++++++++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/edxapp.sql b/edxapp.sql index fbb9d377b1..59202d503e 100644 --- a/edxapp.sql +++ b/edxapp.sql @@ -8816,7 +8816,7 @@ CREATE TABLE `oauth2_provider_application` ( LOCK TABLES `oauth2_provider_application` WRITE; /*!40000 ALTER TABLE `oauth2_provider_application` DISABLE KEYS */; -INSERT INTO `oauth2_provider_application` VALUES (1,'login-service-client-id','','public','password','X4L6BKKf0JgMn1TH0BFnmFLnKRplLYKmUYbqGc2qQLRIGzFjGiwQMPnIOyHMLiW6A8qsVX9mshchCUCcQGLELj5O4eZKCeeEacHON8JikewexicMCCw22gtxwkO4eGPu','Login Service for JWT Cookies',2,0,'2021-07-30 19:58:11.944124','2021-07-30 19:58:11.944166'),(2,'AjEA1P865dsBif42kQYMWJiLLGKQSVSYXy5X6o1V','','confidential','client-credentials','sZuhQuuV3QtpgpXqY6wXrPhB09R81Ijk8CBfzoSP9NER1xYEoLBaqKSggXzvCCjjIKOmytxV6Is3ClqqKMmRZtXTau7r9jrx5dfdgXRCGMSEIFy4xrWOC7Af3F8JP3jY','retirement',9,0,'2021-07-30 20:15:27.098232','2021-07-30 20:15:27.098284'),(3,'ecommerce-sso-key','http://localhost:18130/complete/edx-oauth2/','confidential','authorization-code','ecommerce-sso-secret','ecommerce-sso',1,1,'2021-07-30 20:18:16.419993','2021-07-30 20:18:16.420058'),(4,'ecommerce-backend-service-key','','confidential','client-credentials','ecommerce-backend-service-secret','ecommerce-backend-service',1,0,'2021-07-30 20:18:28.452549','2021-07-30 20:18:28.452606'); +INSERT INTO `oauth2_provider_application` VALUES (1,'login-service-client-id','','public','password','X4L6BKKf0JgMn1TH0BFnmFLnKRplLYKmUYbqGc2qQLRIGzFjGiwQMPnIOyHMLiW6A8qsVX9mshchCUCcQGLELj5O4eZKCeeEacHON8JikewexicMCCw22gtxwkO4eGPu','Login Service for JWT Cookies',2,0,'2021-07-30 19:58:11.944124','2021-07-30 19:58:11.944166'),(2,'AjEA1P865dsBif42kQYMWJiLLGKQSVSYXy5X6o1V','','confidential','client-credentials','sZuhQuuV3QtpgpXqY6wXrPhB09R81Ijk8CBfzoSP9NER1xYEoLBaqKSggXzvCCjjIKOmytxV6Is3ClqqKMmRZtXTau7r9jrx5dfdgXRCGMSEIFy4xrWOC7Af3F8JP3jY','retirement',9,0,'2021-07-30 20:15:27.098232','2021-07-30 20:15:27.098284'),(3,'ecommerce-sso-key','http://localhost:18130/complete/edx-oauth2/ http://ecommerce.pearson.localhost:18130/complete/edx-oauth2/ http://ecommerce.main.localhost:18130/complete/edx-oauth2/','confidential','authorization-code','ecommerce-sso-secret','ecommerce-sso',1,1,'2021-07-30 20:18:16.419993','2021-07-30 20:18:16.420058'),(4,'ecommerce-backend-service-key','','confidential','client-credentials','ecommerce-backend-service-secret','ecommerce-backend-service',1,0,'2021-07-30 20:18:28.452549','2021-07-30 20:18:28.452606'); /*!40000 ALTER TABLE `oauth2_provider_application` ENABLE KEYS */; UNLOCK TABLES; diff --git a/provision-discovery.sh b/provision-discovery.sh index 676e78de22..ce9d75bfc0 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -4,10 +4,14 @@ set -eu -o pipefail set -x ./provision-ida.sh discovery discovery 18381 +# Declaring mysql contianer id to to execute command to update discovery-sso-key +mysql_container=$(make --silent --no-print-directory dev.print-container.mysql57) +docker exec -i "$mysql_container" mysql -uroot edxapp -e "UPDATE oauth2_provider_application SET redirect_uris='http://localhost:18381/complete/edx-oauth2/ http://discovery.pearson.localhost:18381/complete/edx-oauth2/ http://discovery.main.localhost:18381/complete/edx-oauth2/' WHERE client_id='discovery-sso-key'"; docker-compose exec -T discovery bash -e -c 'rm -rf /edx/var/discovery/*' docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --lms-coursemode-api-url "http://edx.devstack.lms:18000/api/course_modes/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/" --studio-url "http://edx.devstack.studio:18010/" --publisher-url "http://edx.devstack.frontend-app-publisher:18400/"' - +docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 2 --site-domain discovery.pearson.localhost:18381 --code PX --name PearsonX --courses-api-url "http://lms.pearson.localhost:18000/api/courses/v1/" --ecommerce-api-url "http://ecommerce.pearson.localhost:18130/api/v2/" --organizations-api-url "http://lms.pearson.localhost:18000/api/organizations/v0/" --lms-url "http://lms.pearson.localhost:18000/"' +docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 3 --site-domain discovery.main.localhost:18381 --code main --name main --courses-api-url "http://lms.main.localhost:18000/api/courses/v1/" --ecommerce-api-url "http://ecommerce.main.localhost:18130/api/v2/" --organizations-api-url "http://lms.main.localhost:18000/api/organizations/v0/" --lms-url "http://lms.main.localhost:18000/"' set +e # FIXME[bash-e]: Bash scripts should use -e -- but this script fails # (after many retries) when trying to talk to ecommerce diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index 85cda52306..c79dff20e9 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -8,6 +8,8 @@ set -x ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' +docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --payment-processors=paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' +docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=2 --site-domain=ecommerce.pearson.localhost:18130 --partner-code=PX --partner-name="PearsonX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://lms.pearson.localhost:18000 --payment-processors=paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://discovery.pearson.localhost:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' +docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=3 --site-domain=ecommerce.main.localhost:18130 --partner-code=main --partner-name="Base" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://lms.main.localhost:18000 --payment-processors=paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://discovery.main.localhost:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-lms.sh b/provision-lms.sh index 964304d102..9b90b8b810 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -30,11 +30,25 @@ docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && echo # Create an enterprise service user for edxapp and give them appropriate permissions ./enterprise/provision.sh +#Create Organizations +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker add_organization PX PearsonX' +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker add_organization demo Demo' + +#Update Site Configurations +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms create_or_update_site_configuration --enabled example.com --configuration {\"course_org_filter\":[\"demo\"]\,\"COURSE_CATALOG_API_URL\":\"http://edx.devstack.discovery:18381/api/v1/\"}' +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms create_or_update_site_configuration --enabled lms.pearson.localhost:18000 --configuration "{\"course_org_filter\":[\"PX\"],\"COURSE_CATALOG_API_URL\":\"http://discovery.pearson.localhost:18381/api/v1/\",\"ECOMMERCE_PUBLIC_URL_ROOT\":\"http://ecommerce.pearson.localhost:18130/\",\"ECOMMERCE_API_URL\":\"http://ecommerce.pearson.localhost:18130/\"}"' +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms create_or_update_site_configuration --enabled lms.main.localhost:18000 --configuration "{\"course_org_filter\":[\"PX\"],\"COURSE_CATALOG_API_URL\":\"http://discovery.main.localhost:18381/api/v1/\",\"ECOMMERCE_PUBLIC_URL_ROOT\":\"http://ecommerce.main.localhost:18130/\",\"ECOMMERCE_API_URL\":\"http://ecommerce.main.localhost:18130/\"}"' + # Enable the LMS-E-Commerce integration docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -docker-compose exec -T lms bash -e -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker create_course split edx@example.com PX demo1 2020 "PX Course 1"' +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker create_course split edx@example.com PX demo2 2020 "PX Course 2"' + +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker create_course split edx@example.com demo demo1 2020 "Demo Course 1"' +docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker create_course split edx@example.com demo demo2 2020 "Demo Course 2"' +docker-compose exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' # Fix missing vendor file by clearing the cache docker-compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' From eb8af55a49a1336b673b7a338089fd87564fdd25 Mon Sep 17 00:00:00 2001 From: alexjmpb Date: Wed, 22 Feb 2023 12:28:08 -0500 Subject: [PATCH 8/8] fix: updating README.rst --- README.rst | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a4e46237f4..7f70dfac8f 100644 --- a/README.rst +++ b/README.rst @@ -234,6 +234,22 @@ This data collection is behind a consent flag, so please help devstack's maintai make metrics-opt-in +For Pearson we also need to configure some hosts in our local hosts file. To make this you would have to open the file `/etc/hosts`, then open the file with your favorite editor and add the following values: + +.. code:: sh + + 127.0.0.1 lms.pearson.localhost + 127.0.0.1 lms.main.localhost + + 127.0.0.1 edx.devstack.ecommerce + 127.0.0.1 ecommerce.pearson.localhost + 127.0.0.1 ecommerce.main.localhost + + 127.0.0.1 edx.devstack.discovery + 127.0.0.1 discovery.pearson.localhost + 127.0.0.1 discovery.main.localhost + + Now that you're up and running, read about the `most common development workflow`_. Usernames and Passwords @@ -274,6 +290,21 @@ is ``edx``. Service List ------------ +Pearson modifications +~~~~~~~~~~~~~~~~~~~~~ + +:: + + For optimization reasons, we decided to disable some services that we are not going to use. These are: + + * insights + * coursegraph + * analyticsapi + * notes + * opensearch12 + + If you want to reactivate these services you can uncomment the definitions on docker-compose.yml, provision.sh, repo.sh, etc + These are the edX services that Devstack can provision, pull, run, attach to, etc. Each service is accessible at ``localhost`` on a specific port. The table below provides links to the homepage, API root, or API docs of each service, @@ -373,7 +404,7 @@ because services are in the process of being upgraded to Elasticsearch 7, but no support Elasticsearch 7 yet. As we complete these migrations, we will update the dependencies of these containers. - +Be sure that you are installing devstack using an environment in Python 3.8. Advanced Configuration Options ------------------------------