Skip to content

Commit

Permalink
refactor: Completely remove Blockstore (#34739)
Browse files Browse the repository at this point in the history 
Blockstore and all of its (experimental) functionality has been replaced with
openedx-learning, aka "Learning Core". This commit uninstalls the now-unused
openedx-blockstore package and removes all dangling references to it.

Note: This also removes the `copy_library_from_v1_to_v2` management command,
which has been broken ever since we switched from Blockstore to Learning Core.

Part of this DEPR: openedx/public-engineering#238
  • Loading branch information
@kdmccormick
kdmccormick authored May 13, 2024
1 parent d3ffb3e commit 15caa97
Show file tree
Hide file tree
Showing 64 changed files with 186 additions and 1,087 deletions.
2 changes: 1 addition & 1 deletion cms/djangoapps/contentstore/config/waffle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
# .. toggle_creation_date: 2020-08-03
# .. toggle_target_removal_date: 2020-12-31
# .. toggle_warning: Also set settings.LIBRARY_AUTHORING_MICROFRONTEND_URL and ENABLE_LIBRARY_AUTHORING_MICROFRONTEND.
# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/COMM/pages/1545011241/BD-14+Blockstore+Powered+Content+Libraries+Taxonomies
# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/OEPM/pages/4106944527/Libraries+Relaunch+Proposal+For+Product+Review
REDIRECT_TO_LIBRARY_AUTHORING_MICROFRONTEND = WaffleFlag(
f'{WAFFLE_NAMESPACE}.library_authoring_mfe', __name__, LOG_PREFIX
)
Expand Down
121 changes: 0 additions & 121 deletions cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py
View file

This file was deleted.

113 changes: 1 addition & 112 deletions cms/djangoapps/contentstore/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from django.contrib.auth import get_user_model
from django.core.exceptions import SuspiciousOperation
from django.core.files import File
from django.db.transaction import atomic
from django.test import RequestFactory
from django.utils.text import get_valid_filename
from edx_django_utils.monitoring import (
Expand All @@ -31,7 +30,7 @@
from olxcleaner.exceptions import ErrorLevel
from olxcleaner.reporting import report_error_summary, report_errors
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
from opaque_keys.edx.locator import LibraryLocator
from organizations.api import add_organization_course, ensure_organization
from organizations.exceptions import InvalidOrganizationException
from organizations.models import Organization, OrganizationCourse
Expand Down Expand Up @@ -66,7 +65,6 @@
from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider
from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks
from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse
from openedx.core.lib.blockstore_api import get_collection
from openedx.core.lib.extract_archive import safe_extractall
from xmodule.contentstore.django import contentstore
from xmodule.course_block import CourseFields
Expand Down Expand Up @@ -900,115 +898,6 @@ def _create_copy_content_task(v2_library_key, v1_library_key):
)


def _create_metadata(v1_library_key, collection_uuid):
"""instansiate an index for the V2 lib in the collection"""

store = modulestore()
v1_library = store.get_library(v1_library_key)
collection = get_collection(collection_uuid).uuid
# To make it easy, all converted libs are complex, meaning they can contain problems, videos, and text
library_type = 'complex'
org = _parse_organization(v1_library.location.library_key.org)
slug = v1_library.location.library_key.library
title = v1_library.display_name
# V1 libraries do not have descriptions.
description = ''
# permssions & license are most restrictive.
allow_public_learning = False
allow_public_read = False
library_license = '' # '' = ALL_RIGHTS_RESERVED
with atomic():
return v2contentlib_api.create_library(
org,
slug,
title,
description,
allow_public_learning,
allow_public_read,
library_license,
library_type,
)


@shared_task(time_limit=30)
@set_code_owner_attribute
def delete_v2_library_from_v1_library(v1_library_key_string, collection_uuid):
"""
For a V1 Library, delete the matching v2 library, where the library is the result of the copy operation
This method relys on _create_metadata failling for LibraryAlreadyExists in order to obtain the v2 slug.
"""
v1_library_key = CourseKey.from_string(v1_library_key_string)
v2_library_key = LibraryLocatorV2.from_string('lib:' + v1_library_key.org + ':' + v1_library_key.course)

try:
v2contentlib_api.delete_library(v2_library_key)
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": v2_library_key,
"status": "SUCCESS",
"msg": None
}
except Exception as error: # lint-amnesty, pylint: disable=broad-except
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": v2_library_key,
"status": "FAILED",
"msg": f"Exception: {v2_library_key} did not delete: {error}"
}


@shared_task(time_limit=30)
@set_code_owner_attribute
def create_v2_library_from_v1_library(v1_library_key_string, collection_uuid):
"""
write the metadata, permissions, and content of a v1 library into a v2 library in the given collection.
"""

v1_library_key = CourseKey.from_string(v1_library_key_string)

LOGGER.info(f"Copy Library task created for library: {v1_library_key}")

try:
v2_library_metadata = _create_metadata(v1_library_key, collection_uuid)

except v2contentlib_api.LibraryAlreadyExists:
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": None,
"status": "FAILED",
"msg": f"Exception: LibraryAlreadyExists {v1_library_key_string} aleady exists"
}

try:
_create_copy_content_task(v2_library_metadata.key, v1_library_key)
except Exception as error: # lint-amnesty, pylint: disable=broad-except
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": str(v2_library_metadata.key),
"status": "FAILED",
"msg":
f"Could not import content from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}"
}

try:
copy_v1_user_roles_into_v2_library(v2_library_metadata.key, v1_library_key)
except Exception as error: # lint-amnesty, pylint: disable=broad-except
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": str(v2_library_metadata.key),
"status": "FAILED",
"msg":
f"Could not copy permissions from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}"
}

return {
"v1_library_id": v1_library_key_string,
"v2_library_id": str(v2_library_metadata.key),
"status": "SUCCESS",
"msg": None
}


@shared_task(time_limit=30)
@set_code_owner_attribute
def delete_v1_library(v1_library_key_string):
Expand Down
45 changes: 7 additions & 38 deletions cms/envs/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,6 @@
ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_SECRET,
ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL,

# Blockstore
BUNDLE_ASSET_STORAGE_SETTINGS,

# Methods to derive settings
_make_mako_template_dirs,
_make_locale_paths,
Expand Down Expand Up @@ -444,7 +441,7 @@
# .. toggle_use_cases: temporary
# .. toggle_creation_date: 2020-06-20
# .. toggle_target_removal_date: 2020-12-31
# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/COMM/pages/1545011241/BD-14+Blockstore+Powered+Content+Libraries+Taxonomies
# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/OEPM/pages/4106944527/Libraries+Relaunch+Proposal+For+Product+Review
# .. toggle_warning: Also set settings.LIBRARY_AUTHORING_MICROFRONTEND_URL and see
# REDIRECT_TO_LIBRARY_AUTHORING_MICROFRONTEND for rollout.
'ENABLE_LIBRARY_AUTHORING_MICROFRONTEND': False,
Expand Down Expand Up @@ -1033,6 +1030,11 @@
# Paths to wrapper methods which should be applied to every XBlock's FieldData.
XBLOCK_FIELD_DATA_WRAPPERS = ()

# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE
# .. setting_default: default
# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks.
XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default'

############################ ORA 2 ############################################

# By default, don't use a file prefix
Expand Down Expand Up @@ -1690,7 +1692,7 @@
'cms.djangoapps.xblock_config.apps.XBlockConfig',
'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig',

# New (Blockstore-based) XBlock runtime
# New (Learning-Core-based) XBlock runtime
'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig',

# Maintenance tools
Expand Down Expand Up @@ -1873,9 +1875,6 @@
# For edx ace template tags
'edx_ace',

# Blockstore
'blockstore.apps.bundles',

# alternative swagger generator for CMS API
'drf_spectacular',

Expand Down Expand Up @@ -2243,25 +2242,11 @@

DATABASE_ROUTERS = [
'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter',
'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter',
]

############################ Cache Configuration ###############################

CACHES = {
'blockstore': {
'KEY_PREFIX': 'blockstore',
'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key',
'LOCATION': ['localhost:11211'],
'TIMEOUT': '86400', # This data should be long-lived for performance, BundleCache handles invalidation
'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
'OPTIONS': {
'no_delay': True,
'ignore_exc': True,
'use_pooling': True,
'connect_timeout': 0.5
}
},
'course_structure_cache': {
'KEY_PREFIX': 'course_structure',
'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key',
Expand Down Expand Up @@ -2699,22 +2684,6 @@

PROCTORING_SETTINGS = {}

################## BLOCKSTORE RELATED SETTINGS #########################

# Which of django's caches to use for storing anonymous user state for XBlocks
# in the blockstore-based XBlock runtime
XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default'

# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT
# .. setting_default: 3000
# .. setting_description: Maximum time-to-live of cached Bundles fetched from
# Blockstore, in seconds. When the values returned from Blockstore have
# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache
# must be lower than the minimum TTL of those values.
# We use a default of 3000s (50mins) because temporary URLs are often
# configured to expire after one hour.
BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000

###################### LEARNER PORTAL ################################
LEARNER_PORTAL_URL_ROOT = 'https://learner-portal-localhost:18000'

Expand Down
3 changes: 0 additions & 3 deletions cms/envs/devstack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,9 +218,6 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing

ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL = "http://edx.devstack.lms/oauth2"

############################### BLOCKSTORE #####################################
BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/"

#####################################################################

# pylint: disable=wrong-import-order, wrong-import-position
Expand Down
5 changes: 0 additions & 5 deletions cms/envs/production.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -405,11 +405,6 @@ def get_env_setting(setting):
CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE)
DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG', DOC_STORE_CONFIG)

############################### BLOCKSTORE #####################################
BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/"
# Configure an API auth token at (blockstore URL)/admin/authtoken/token/
BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None)

# Celery Broker
CELERY_ALWAYS_EAGER = ENV_TOKENS.get("CELERY_ALWAYS_EAGER", False)
CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")
Expand Down
Loading

0 comments on commit 15caa97

Please sign in to comment.