From c6d3954e7bfd34715c0aea41aacb98666a0c6c61 Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Mon, 4 Nov 2024 13:25:51 +0500 Subject: [PATCH 01/13] fix: migrations to make postgresql compatible. --- .../migrations/0001_initial.py | 31 +++++++++++-- .../0001_data__add_ecommerce_service_user.py | 14 +++--- .../migrations/0011_csm_id_bigint.py | 21 ++++++--- ...oricalpersistentsubsectiongradeoverride.py | 1 + .../migrations/0009_readd_facebook_url.py | 45 +++++++++---------- .../migrations/0001_initial.py | 31 +++++++------ requirements/edx/kernel.in | 1 + 7 files changed, 90 insertions(+), 54 deletions(-) diff --git a/common/djangoapps/split_modulestore_django/migrations/0001_initial.py b/common/djangoapps/split_modulestore_django/migrations/0001_initial.py index 25b752da68d8..46f3381044ba 100644 --- a/common/djangoapps/split_modulestore_django/migrations/0001_initial.py +++ b/common/djangoapps/split_modulestore_django/migrations/0001_initial.py @@ -1,15 +1,40 @@ # Generated by Django 2.2.20 on 2021-05-07 18:29, manually modified to make "course_id" column case sensitive from django.conf import settings -from django.db import migrations, models +from django.db import migrations, models, connection import django.db.models.deletion import opaque_keys.edx.django.models import simple_history.models +def generate_split_module_sql(db_engine): + if 'mysql' in db_engine: + return 'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;' + elif 'postgresql' in db_engine: + return """ + ALTER TABLE split_modulestore_django_splitmodulestorecourseindex + ALTER COLUMN course_id TYPE VARCHAR(255), + ALTER COLUMN course_id SET NOT NULL; + + ALTER TABLE split_modulestore_django_splitmodulestorecourseindex + ADD CONSTRAINT course_id_unique UNIQUE (course_id); + """ + + +def generate_split_history_module_sql(db_engine): + if 'mysql' in db_engine: + return 'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;' + elif 'postgresql' in db_engine: + return """ + ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex + ALTER COLUMN course_id TYPE VARCHAR(255), + ALTER COLUMN course_id SET NOT NULL, + ALTER COLUMN course_id SET DATA TYPE VARCHAR(255) COLLATE "C"; + """ class Migration(migrations.Migration): initial = True + db_engine = connection.settings_dict['ENGINE'] dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), @@ -65,11 +90,11 @@ class Migration(migrations.Migration): # Custom code: Convert columns to utf8_bin because we want to allow # case-sensitive comparisons for CourseKeys, which were case-sensitive in MongoDB migrations.RunSQL( - 'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;', + generate_split_module_sql(db_engine), reverse_sql=migrations.RunSQL.noop, ), migrations.RunSQL( - 'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;', + generate_split_history_module_sql(db_engine), reverse_sql=migrations.RunSQL.noop, ), ] diff --git a/lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py b/lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py index bc83ce2288b5..9deb9dac4a7c 100644 --- a/lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py +++ b/lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py @@ -1,7 +1,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.models import User -from django.db import migrations, models +from django.db import migrations, models, transaction USERNAME = settings.ECOMMERCE_SERVICE_WORKER_USERNAME EMAIL = USERNAME + '@fake.email' @@ -9,14 +9,16 @@ def forwards(apps, schema_editor): """Add the service user.""" User = get_user_model() - user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL) - if created: - user.set_unusable_password() - user.save() + with transaction.atomic(): + user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL) + if created: + user.set_unusable_password() + user.save() def backwards(apps, schema_editor): """Remove the service user.""" - User.objects.get(username=USERNAME, email=EMAIL).delete() + with transaction.atomic(): + User.objects.get(username=USERNAME, email=EMAIL).delete() class Migration(migrations.Migration): diff --git a/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py b/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py index abcebcf0b45f..06a2ebd44265 100644 --- a/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py +++ b/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py @@ -1,6 +1,5 @@ # Generated by Django 1.11.23 on 2019-08-28 15:50 - import lms.djangoapps.courseware.fields from django.conf import settings @@ -10,22 +9,33 @@ class CsmBigInt(AlterField): ''' Subclass AlterField migration class to split SQL between two different databases - We can't use the normal AlterField migration operation because Django generate and routes migrations at the model + We can't use the normal AlterField migration operation because Django generates and routes migrations at the model level and the coursewarehistoryextended_studentmodulehistoryextended table is in a different database ''' def database_forwards(self, app_label, schema_editor, from_state, to_state): if hasattr(schema_editor.connection, 'is_in_memory_db') and schema_editor.connection.is_in_memory_db(): # sqlite3 doesn't support 'MODIFY', so skipping during tests return + to_model = to_state.apps.get_model(app_label, self.model_name) + if schema_editor.connection.alias == 'student_module_history': if settings.FEATURES["ENABLE_CSMH_EXTENDED"]: - schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;") + if schema_editor.connection.vendor == 'mysql': + schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;") + elif schema_editor.connection.vendor == 'postgresql': + schema_editor.execute("ALTER TABLE coursewarehistoryextended_studentmodulehistoryextended ALTER COLUMN student_module_id TYPE bigint;") elif self.allow_migrate_model(schema_editor.connection.alias, to_model): - schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;") + if schema_editor.connection.vendor == 'postgresql': + # For PostgreSQL + schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET DATA TYPE bigint;") + schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET NOT NULL;") + else: + # For MySQL + schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;") def database_backwards(self, app_label, schema_editor, from_state, to_state): - # Make backwards migration a no-op, app will still work if column is wider than expected + # Make backwards migration a no-op; app will still work if column is wider than expected pass class Migration(migrations.Migration): @@ -33,6 +43,7 @@ class Migration(migrations.Migration): dependencies = [ ('courseware', '0010_auto_20190709_1559'), ] + if settings.FEATURES["ENABLE_CSMH_EXTENDED"]: dependencies.append(('coursewarehistoryextended', '0002_force_studentmodule_index')) diff --git a/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py b/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py index 75522833ae54..b5050efb71cc 100644 --- a/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py +++ b/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py @@ -27,6 +27,7 @@ class Migration(migrations.Migration): ('possible_graded_override', models.FloatField(blank=True, null=True)), ('history_id', models.AutoField(primary_key=True, serialize=False)), ('history_date', models.DateTimeField()), + ('grade_id', models.IntegerField()), ('history_change_reason', models.CharField(max_length=100, null=True)), ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), ('grade', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='grades.PersistentSubsectionGrade')), diff --git a/openedx/core/djangoapps/content/course_overviews/migrations/0009_readd_facebook_url.py b/openedx/core/djangoapps/content/course_overviews/migrations/0009_readd_facebook_url.py index dc99d79731ef..37bd76548e9e 100644 --- a/openedx/core/djangoapps/content/course_overviews/migrations/0009_readd_facebook_url.py +++ b/openedx/core/djangoapps/content/course_overviews/migrations/0009_readd_facebook_url.py @@ -1,45 +1,42 @@ from django.db import migrations, models, connection def table_description(): - """Handle Mysql/Pg vs Sqlite""" - # django's mysql/pg introspection.get_table_description tries to select * - # from table and fails during initial migrations from scratch. - # sqlite does not have this failure, so we can use the API. - # For not-sqlite, query information-schema directly with code lifted - # from the internals of django.db.backends.mysql.introspection.py - + """Handle MySQL/Postgres vs SQLite compatibility for table introspection""" if connection.vendor == 'sqlite': fields = connection.introspection.get_table_description(connection.cursor(), 'course_overviews_courseoverview') return [f.name for f in fields] else: cursor = connection.cursor() - cursor.execute(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = 'course_overviews_courseoverview' AND table_schema = DATABASE()""") + if connection.vendor == 'mysql': + cursor.execute(""" + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'course_overviews_courseoverview' AND table_schema = DATABASE() + """) + elif connection.vendor == 'postgresql': + cursor.execute(""" + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'course_overviews_courseoverview' AND table_catalog = current_database() + """) rows = cursor.fetchall() return [r[0] for r in rows] - class Migration(migrations.Migration): dependencies = [ ('course_overviews', '0008_remove_courseoverview_facebook_url'), ] - # An original version of 0008 removed the facebook_url field We need to - # handle the case where our noop 0008 ran AND the case where the original - # 0008 ran. We do that by using the standard information_schema to find out - # what columns exist. _meta is unavailable as the column has already been - # removed from the model operations = [] fields = table_description() - # during a migration from scratch, fields will be empty, but we do not want to add - # an additional facebook_url + # Ensure 'facebook_url' is added if it does not exist in the table if fields and not any(f == 'facebook_url' for f in fields): - operations += migrations.AddField( - model_name='courseoverview', - name='facebook_url', - field=models.TextField(null=True), - ), + operations.append( + migrations.AddField( + model_name='courseoverview', + name='facebook_url', + field=models.TextField(null=True), + ) + ) diff --git a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py index 22aad21ffb27..6e93f875a083 100644 --- a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py +++ b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py @@ -8,8 +8,20 @@ import django.utils.timezone import model_utils.fields import opaque_keys.edx.django.models +from django.conf import settings +from django.db import connection - +def run_before_migrate(apps, schema_editor): + if connection.vendor == 'mysql': + # MySQL: utf8_bin collation + schema_editor.execute('ALTER TABLE learning_sequences_learningcontext MODIFY context_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;') + schema_editor.execute('ALTER TABLE learning_sequences_coursesection MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;') + schema_editor.execute('ALTER TABLE learning_sequences_learningsequence MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;') + elif connection.vendor == 'postgresql': + # PostgreSQL: Use binary collation + schema_editor.execute('ALTER TABLE learning_sequences_learningcontext ALTER COLUMN context_key TYPE VARCHAR(255) COLLATE "C";') + schema_editor.execute('ALTER TABLE learning_sequences_coursesection ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";') + schema_editor.execute('ALTER TABLE learning_sequences_learningsequence ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";') class Migration(migrations.Migration): initial = True @@ -106,19 +118,6 @@ class Migration(migrations.Migration): index_together={('learning_context', 'ordering')}, ), - # Custom code: Convert columns to utf8_bin because we want to allow - # case-sensitive comparisons for things like UsageKeys, CourseKeys, and - # slugs. - migrations.RunSQL( - 'ALTER TABLE learning_sequences_learningcontext MODIFY context_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;', - reverse_sql=migrations.RunSQL.noop, - ), - migrations.RunSQL( - 'ALTER TABLE learning_sequences_coursesection MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;', - reverse_sql=migrations.RunSQL.noop, - ), - migrations.RunSQL( - 'ALTER TABLE learning_sequences_learningsequence MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;', - reverse_sql=migrations.RunSQL.noop, - ), + # Custom code: Convert columns to utf8_bin for MySQL or the equivalent for PostgreSQL + migrations.RunPython(run_before_migrate), ] diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 7323c243accf..bc8563cdc4f4 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -160,3 +160,4 @@ webob web-fragments # Provides the ability to render fragments of web pages XBlock[django] # Courseware component architecture xss-utils # https://github.com/openedx/edx-platform/pull/20633 Fix XSS via Translations +psycopg2-binary From 5fdf2e521dbf9b1bcf4c3ac23559e56a377b238f Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Mon, 4 Nov 2024 14:00:20 +0500 Subject: [PATCH 02/13] fix: duplicate column issue --- ...oricalpersistentsubsectiongradeoverride.py | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py b/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py index b5050efb71cc..0fbf8d1f5974 100644 --- a/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py +++ b/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py @@ -2,13 +2,24 @@ from django.conf import settings -from django.db import migrations, models +from django.db import migrations, models, connection import django.db.models.deletion import simple_history.models -class Migration(migrations.Migration): +def get_compatible_columns(): + columns = [ + ('grade', + models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, + related_name='+', to='grades.PersistentSubsectionGrade')), + ] + if connection.vendor == 'postgresql': + columns = [('grade_id', models.IntegerField()), ] + columns + + return columns + +class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('grades', '0014_persistentsubsectiongradeoverridehistory'), @@ -18,21 +29,22 @@ class Migration(migrations.Migration): migrations.CreateModel( name='HistoricalPersistentSubsectionGradeOverride', fields=[ - ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), - ('created', models.DateTimeField(blank=True, db_index=True, editable=False)), - ('modified', models.DateTimeField(blank=True, db_index=True, editable=False)), - ('earned_all_override', models.FloatField(blank=True, null=True)), - ('possible_all_override', models.FloatField(blank=True, null=True)), - ('earned_graded_override', models.FloatField(blank=True, null=True)), - ('possible_graded_override', models.FloatField(blank=True, null=True)), - ('history_id', models.AutoField(primary_key=True, serialize=False)), - ('history_date', models.DateTimeField()), - ('grade_id', models.IntegerField()), - ('history_change_reason', models.CharField(max_length=100, null=True)), - ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), - ('grade', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='grades.PersistentSubsectionGrade')), - ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), - ], + ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), + ('created', models.DateTimeField(blank=True, db_index=True, editable=False)), + ('modified', models.DateTimeField(blank=True, db_index=True, editable=False)), + ('earned_all_override', models.FloatField(blank=True, null=True)), + ('possible_all_override', models.FloatField(blank=True, null=True)), + ('earned_graded_override', models.FloatField(blank=True, null=True)), + ('possible_graded_override', models.FloatField(blank=True, null=True)), + ('history_id', models.AutoField(primary_key=True, serialize=False)), + ('history_date', models.DateTimeField()), + ('history_change_reason', models.CharField(max_length=100, null=True)), + ('history_type', + models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), + ('history_user', + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', + to=settings.AUTH_USER_MODEL)), + ] + get_compatible_columns(), options={ 'ordering': ('-history_date', '-history_id'), 'get_latest_by': 'history_date', From da1d4ede564cfe4e63e67024667710cf88509edf Mon Sep 17 00:00:00 2001 From: Muhammad Qasim Gulzar <19186089+qasimgulzar@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:30:23 +0500 Subject: [PATCH 03/13] fix: learning sequences migration for mysql --- .../migrations/0001_initial.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py index 6e93f875a083..0346303d01ea 100644 --- a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py +++ b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py @@ -11,17 +11,17 @@ from django.conf import settings from django.db import connection -def run_before_migrate(apps, schema_editor): +def run_before_migrate(apps, migrations): if connection.vendor == 'mysql': # MySQL: utf8_bin collation - schema_editor.execute('ALTER TABLE learning_sequences_learningcontext MODIFY context_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;') - schema_editor.execute('ALTER TABLE learning_sequences_coursesection MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;') - schema_editor.execute('ALTER TABLE learning_sequences_learningsequence MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;') + return [migrations.RunSQL('ALTER TABLE learning_sequences_learningcontext MODIFY context_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;',reverse_sql=migrations.RunSQL.noop,), + migrations.RunSQL('ALTER TABLE learning_sequences_coursesection MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;',reverse_sql=migrations.RunSQL.noop,), + migrations.RunSQL('ALTER TABLE learning_sequences_learningsequence MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;',reverse_sql=migrations.RunSQL.noop,)] elif connection.vendor == 'postgresql': # PostgreSQL: Use binary collation - schema_editor.execute('ALTER TABLE learning_sequences_learningcontext ALTER COLUMN context_key TYPE VARCHAR(255) COLLATE "C";') - schema_editor.execute('ALTER TABLE learning_sequences_coursesection ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";') - schema_editor.execute('ALTER TABLE learning_sequences_learningsequence ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";') + return [migrations.RunSQL('ALTER TABLE learning_sequences_learningcontext ALTER COLUMN context_key TYPE VARCHAR(255) COLLATE "C";',reverse_sql=migrations.RunSQL.noop,), + migrations.RunSQL('ALTER TABLE learning_sequences_coursesection ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";',reverse_sql=migrations.RunSQL.noop,), + migrations.RunSQL('ALTER TABLE learning_sequences_learningsequence ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";',reverse_sql=migrations.RunSQL.noop,)] class Migration(migrations.Migration): initial = True @@ -117,7 +117,4 @@ class Migration(migrations.Migration): name='coursesection', index_together={('learning_context', 'ordering')}, ), - - # Custom code: Convert columns to utf8_bin for MySQL or the equivalent for PostgreSQL - migrations.RunPython(run_before_migrate), - ] + ]+run_before_migrate(migrations) From bb8cf5d28980c34af67049930bcf9a42f5556a55 Mon Sep 17 00:00:00 2001 From: Muhammad Qasim Gulzar <19186089+qasimgulzar@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:40:53 +0500 Subject: [PATCH 04/13] fix: mysql specific change --- .../content/learning_sequences/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py index 0346303d01ea..171fdfee03c9 100644 --- a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py +++ b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py @@ -11,7 +11,7 @@ from django.conf import settings from django.db import connection -def run_before_migrate(apps, migrations): +def run_before_migrate(migrations): if connection.vendor == 'mysql': # MySQL: utf8_bin collation return [migrations.RunSQL('ALTER TABLE learning_sequences_learningcontext MODIFY context_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;',reverse_sql=migrations.RunSQL.noop,), @@ -117,4 +117,4 @@ class Migration(migrations.Migration): name='coursesection', index_together={('learning_context', 'ordering')}, ), - ]+run_before_migrate(migrations) + ] + run_before_migrate(migrations) From bd32cf2015aef350846dda377f0f25b7ebb6ed42 Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Tue, 5 Nov 2024 13:14:25 +0500 Subject: [PATCH 05/13] fix: compile dependencies --- requirements/edx/base.txt | 2 ++ requirements/edx/development.txt | 4 ++++ requirements/edx/doc.txt | 2 ++ requirements/edx/testing.txt | 2 ++ 4 files changed, 10 insertions(+) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c697f8eb81fb..dfbfe3e0d1ec 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -900,6 +900,8 @@ psutil==6.1.0 # via # -r requirements/edx/paver.txt # edx-django-utils +psycopg2-binary==2.9.10 + # via -r requirements/edx/kernel.in py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # via # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index a07b24914d2d..9527ef9bfb49 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1526,6 +1526,10 @@ psutil==6.1.0 # edx-django-utils # pact-python # pytest-xdist +psycopg2-binary==2.9.10 + # via + # -r requirements/edx/doc.txt + # -r requirements/edx/testing.txt py==1.11.0 # via -r requirements/edx/testing.txt py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index fb3d610ded6e..764475a7a10a 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1090,6 +1090,8 @@ psutil==6.1.0 # via # -r requirements/edx/base.txt # edx-django-utils +psycopg2-binary==2.9.10 + # via -r requirements/edx/base.txt py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # via # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 6984a56265da..2ba5b4a3b82f 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1149,6 +1149,8 @@ psutil==6.1.0 # edx-django-utils # pact-python # pytest-xdist +psycopg2-binary==2.9.10 + # via -r requirements/edx/base.txt py==1.11.0 # via -r requirements/edx/testing.in py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz From 039472e82692a345ee1e8833a51c0b2efee60bdf Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Tue, 5 Nov 2024 14:29:47 +0500 Subject: [PATCH 06/13] fix: migration issue --- ...oricalpersistentsubsectiongradeoverride.py | 67 +++++++++---------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py b/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py index 0fbf8d1f5974..8670853f9acf 100644 --- a/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py +++ b/lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py @@ -1,22 +1,10 @@ # Generated by Django 1.11.20 on 2019-06-05 13:59 -from django.conf import settings -from django.db import migrations, models, connection import django.db.models.deletion import simple_history.models - - -def get_compatible_columns(): - columns = [ - ('grade', - models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, - related_name='+', to='grades.PersistentSubsectionGrade')), - ] - if connection.vendor == 'postgresql': - columns = [('grade_id', models.IntegerField()), ] + columns - - return columns +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): @@ -29,27 +17,34 @@ class Migration(migrations.Migration): migrations.CreateModel( name='HistoricalPersistentSubsectionGradeOverride', fields=[ - ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), - ('created', models.DateTimeField(blank=True, db_index=True, editable=False)), - ('modified', models.DateTimeField(blank=True, db_index=True, editable=False)), - ('earned_all_override', models.FloatField(blank=True, null=True)), - ('possible_all_override', models.FloatField(blank=True, null=True)), - ('earned_graded_override', models.FloatField(blank=True, null=True)), - ('possible_graded_override', models.FloatField(blank=True, null=True)), - ('history_id', models.AutoField(primary_key=True, serialize=False)), - ('history_date', models.DateTimeField()), - ('history_change_reason', models.CharField(max_length=100, null=True)), - ('history_type', - models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), - ('history_user', - models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', - to=settings.AUTH_USER_MODEL)), - ] + get_compatible_columns(), - options={ - 'ordering': ('-history_date', '-history_id'), - 'get_latest_by': 'history_date', - 'verbose_name': 'historical persistent subsection grade override', - }, - bases=(simple_history.models.HistoricalChanges, models.Model), + ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), + ('created', models.DateTimeField(blank=True, db_index=True, editable=False)), + ('modified', models.DateTimeField(blank=True, db_index=True, editable=False)), + ('earned_all_override', models.FloatField(blank=True, null=True)), + ('possible_all_override', models.FloatField(blank=True, null=True)), + ('earned_graded_override', models.FloatField(blank=True, null=True)), + ('possible_graded_override', models.FloatField(blank=True, null=True)), + ('history_id', models.AutoField(primary_key=True, serialize=False)), + ('history_date', models.DateTimeField()), + ('history_change_reason', models.CharField(max_length=100, null=True)), + ('history_type', + models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), + ('history_user', + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', + to=settings.AUTH_USER_MODEL)), + ], + options = { + 'ordering': ('-history_date', '-history_id'), + 'get_latest_by': 'history_date', + 'verbose_name': 'historical persistent subsection grade override', + }, + bases = (simple_history.models.HistoricalChanges, models.Model), + ), + migrations.AddField( + model_name='historicalpersistentsubsectiongradeoverride', + name='grade', + field=models.ForeignKey(blank=True, db_constraint=False, null=True, + on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', + to='grades.PersistentSubsectionGrade'), ), ] From bea430f3c66b4e34aea7dec333f9525d7ff38f75 Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Tue, 5 Nov 2024 16:22:27 +0500 Subject: [PATCH 07/13] fix: broken test --- .../migrations/0001_initial.py | 245 ++++++++++-------- 1 file changed, 143 insertions(+), 102 deletions(-) diff --git a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py index 171fdfee03c9..665a8082d979 100644 --- a/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py +++ b/openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py @@ -3,118 +3,159 @@ # Manually modified to collate some fields as utf8_bin for case sensitive # matching. -from django.db import migrations, models -import django.db.models.deletion import django.utils.timezone import model_utils.fields import opaque_keys.edx.django.models -from django.conf import settings from django.db import connection +from django.db import migrations, models -def run_before_migrate(migrations): - if connection.vendor == 'mysql': - # MySQL: utf8_bin collation - return [migrations.RunSQL('ALTER TABLE learning_sequences_learningcontext MODIFY context_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;',reverse_sql=migrations.RunSQL.noop,), - migrations.RunSQL('ALTER TABLE learning_sequences_coursesection MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;',reverse_sql=migrations.RunSQL.noop,), - migrations.RunSQL('ALTER TABLE learning_sequences_learningsequence MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;',reverse_sql=migrations.RunSQL.noop,)] - elif connection.vendor == 'postgresql': + +def run_before_migrate(migrations, db_engine): + if 'postgresql' in db_engine: # PostgreSQL: Use binary collation - return [migrations.RunSQL('ALTER TABLE learning_sequences_learningcontext ALTER COLUMN context_key TYPE VARCHAR(255) COLLATE "C";',reverse_sql=migrations.RunSQL.noop,), - migrations.RunSQL('ALTER TABLE learning_sequences_coursesection ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";',reverse_sql=migrations.RunSQL.noop,), - migrations.RunSQL('ALTER TABLE learning_sequences_learningsequence ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";',reverse_sql=migrations.RunSQL.noop,)] -class Migration(migrations.Migration): + return [ + migrations.RunSQL( + 'ALTER TABLE learning_sequences_learningcontext ALTER COLUMN context_key TYPE VARCHAR(255) COLLATE "C";', + reverse_sql=migrations.RunSQL.noop, ), + migrations.RunSQL( + 'ALTER TABLE learning_sequences_coursesection ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";', + reverse_sql=migrations.RunSQL.noop, ), + migrations.RunSQL( + 'ALTER TABLE learning_sequences_learningsequence ALTER COLUMN usage_key TYPE VARCHAR(255) COLLATE "C";', + reverse_sql=migrations.RunSQL.noop, ), + ] + return [ + migrations.RunSQL( + 'ALTER TABLE learning_sequences_learningcontext MODIFY context_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;', + reverse_sql=migrations.RunSQL.noop, ), + migrations.RunSQL( + 'ALTER TABLE learning_sequences_coursesection MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;', + reverse_sql=migrations.RunSQL.noop, ), + migrations.RunSQL( + 'ALTER TABLE learning_sequences_learningsequence MODIFY usage_key VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin;', + reverse_sql=migrations.RunSQL.noop, ), + ] + + +class Migration(migrations.Migration): initial = True + db_engine = connection.settings_dict['ENGINE'] dependencies = [ ] operations = [ - migrations.CreateModel( - name='CourseSection', - fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('ordering', models.PositiveIntegerField()), - ('usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), - ('title', models.CharField(max_length=1000)), - ('hide_from_toc', models.BooleanField(default=False)), - ('visible_to_staff_only', models.BooleanField(default=False)), - ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), - ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), - ], - ), - migrations.CreateModel( - name='CourseSectionSequence', - fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('ordering', models.PositiveIntegerField()), - ('hide_from_toc', models.BooleanField(default=False)), - ('visible_to_staff_only', models.BooleanField(default=False)), - ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), - ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), - ], - ), - migrations.CreateModel( - name='LearningContext', - fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('context_key', opaque_keys.edx.django.models.LearningContextKeyField(db_index=True, max_length=255, unique=True)), - ('title', models.CharField(max_length=255)), - ('published_at', models.DateTimeField()), - ('published_version', models.CharField(max_length=255)), - ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), - ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), - ], - ), - migrations.CreateModel( - name='LearningSequence', - fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('learning_context', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sequences', to='learning_sequences.LearningContext')), - ('usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), - ('title', models.CharField(max_length=1000)), - ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), - ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), - ], - ), - migrations.AddIndex( - model_name='learningcontext', - index=models.Index(fields=['-published_at'], name='learning_se_publish_62319b_idx'), - ), - migrations.AddField( - model_name='coursesectionsequence', - name='learning_context', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='section_sequences', to='learning_sequences.LearningContext'), - ), - migrations.AddField( - model_name='coursesectionsequence', - name='section', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='learning_sequences.CourseSection'), - ), - migrations.AddField( - model_name='coursesectionsequence', - name='sequence', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='learning_sequences.LearningSequence'), - ), - migrations.AddField( - model_name='coursesection', - name='learning_context', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sections', to='learning_sequences.LearningContext'), - ), - migrations.AlterUniqueTogether( - name='learningsequence', - unique_together={('learning_context', 'usage_key')}, - ), - migrations.AlterUniqueTogether( - name='coursesectionsequence', - unique_together={('learning_context', 'ordering')}, - ), - migrations.AlterUniqueTogether( - name='coursesection', - unique_together={('learning_context', 'usage_key')}, - ), - migrations.AlterIndexTogether( - name='coursesection', - index_together={('learning_context', 'ordering')}, - ), - ] + run_before_migrate(migrations) + migrations.CreateModel( + name='CourseSection', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('ordering', models.PositiveIntegerField()), + ('usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), + ('title', models.CharField(max_length=1000)), + ('hide_from_toc', models.BooleanField(default=False)), + ('visible_to_staff_only', models.BooleanField(default=False)), + ('created', + model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, + verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, + editable=False, + verbose_name='modified')), + ], + ), + migrations.CreateModel( + name='CourseSectionSequence', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('ordering', models.PositiveIntegerField()), + ('hide_from_toc', models.BooleanField(default=False)), + ('visible_to_staff_only', models.BooleanField(default=False)), + ('created', + model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, + verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, + editable=False, + verbose_name='modified')), + ], + ), + migrations.CreateModel( + name='LearningContext', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('context_key', + opaque_keys.edx.django.models.LearningContextKeyField(db_index=True, max_length=255, + unique=True)), + ('title', models.CharField(max_length=255)), + ('published_at', models.DateTimeField()), + ('published_version', models.CharField(max_length=255)), + ('created', + model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, + verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, + editable=False, + verbose_name='modified')), + ], + ), + migrations.CreateModel( + name='LearningSequence', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('learning_context', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sequences', + to='learning_sequences.LearningContext')), + ('usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), + ('title', models.CharField(max_length=1000)), + ('created', + model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, + verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, + editable=False, + verbose_name='modified')), + ], + ), + migrations.AddIndex( + model_name='learningcontext', + index=models.Index(fields=['-published_at'], name='learning_se_publish_62319b_idx'), + ), + migrations.AddField( + model_name='coursesectionsequence', + name='learning_context', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, + related_name='section_sequences', + to='learning_sequences.LearningContext'), + ), + migrations.AddField( + model_name='coursesectionsequence', + name='section', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, + to='learning_sequences.CourseSection'), + ), + migrations.AddField( + model_name='coursesectionsequence', + name='sequence', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, + to='learning_sequences.LearningSequence'), + ), + migrations.AddField( + model_name='coursesection', + name='learning_context', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sections', + to='learning_sequences.LearningContext'), + ), + migrations.AlterUniqueTogether( + name='learningsequence', + unique_together={('learning_context', 'usage_key')}, + ), + migrations.AlterUniqueTogether( + name='coursesectionsequence', + unique_together={('learning_context', 'ordering')}, + ), + migrations.AlterUniqueTogether( + name='coursesection', + unique_together={('learning_context', 'usage_key')}, + ), + migrations.AlterIndexTogether( + name='coursesection', + index_together={('learning_context', 'ordering')}, + ), + ] + run_before_migrate(migrations, db_engine=db_engine) From 5f0fc67328ecc511abddb42807e35531e85d99be Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Mon, 18 Nov 2024 13:40:18 +0500 Subject: [PATCH 08/13] fix: using models.BigAutoField instead of custom field. --- lms/djangoapps/courseware/fields.py | 33 ------------------- .../migrations/0011_csm_id_bigint.py | 4 +-- lms/djangoapps/courseware/models.py | 3 +- .../migrations/0001_initial.py | 3 +- .../coursewarehistoryextended/models.py | 3 +- .../grades/migrations/0001_initial.py | 4 +-- .../0006_persistent_course_grades.py | 4 +-- lms/djangoapps/grades/models.py | 5 ++- 8 files changed, 9 insertions(+), 50 deletions(-) diff --git a/lms/djangoapps/courseware/fields.py b/lms/djangoapps/courseware/fields.py index 7929ae5efe94..55c20f948120 100644 --- a/lms/djangoapps/courseware/fields.py +++ b/lms/djangoapps/courseware/fields.py @@ -1,36 +1,3 @@ """ Custom fields """ - - -from django.db.models.fields import AutoField - - -class UnsignedBigIntAutoField(AutoField): - """ - An unsigned 8-byte integer for auto-incrementing primary keys. - """ - def db_type(self, connection): - if connection.settings_dict['ENGINE'] == 'django.db.backends.mysql': - return "bigint UNSIGNED AUTO_INCREMENT" - elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3': - # Sqlite will only auto-increment the ROWID column. Any INTEGER PRIMARY KEY column - # is an alias for that (https://www.sqlite.org/autoinc.html). An unsigned integer - # isn't an alias for ROWID, so we have to give up on the unsigned part. - return "integer" - elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2': - # Pg's bigserial is implicitly unsigned (doesn't allow negative numbers) and - # goes 1-9.2x10^18 - return "BIGSERIAL" - else: - return None - - def rel_db_type(self, connection): - if connection.settings_dict['ENGINE'] == 'django.db.backends.mysql': - return "bigint UNSIGNED" - elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3': - return "integer" - elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2': - return "BIGSERIAL" - else: - return None diff --git a/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py b/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py index 06a2ebd44265..8dae3c7be92f 100644 --- a/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py +++ b/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py @@ -3,7 +3,7 @@ import lms.djangoapps.courseware.fields from django.conf import settings -from django.db import migrations +from django.db import migrations, models from django.db.migrations import AlterField class CsmBigInt(AlterField): @@ -51,6 +51,6 @@ class Migration(migrations.Migration): CsmBigInt( model_name='studentmodule', name='id', - field=lms.djangoapps.courseware.fields.UnsignedBigIntAutoField(primary_key=True, serialize=False), + field=models.BigAutoField(primary_key=True), ) ] diff --git a/lms/djangoapps/courseware/models.py b/lms/djangoapps/courseware/models.py index eacf2424de6c..8ec4c164a202 100644 --- a/lms/djangoapps/courseware/models.py +++ b/lms/djangoapps/courseware/models.py @@ -28,7 +28,6 @@ from edx_django_utils.cache.utils import RequestCache from model_utils.models import TimeStampedModel from opaque_keys.edx.django.models import BlockTypeKeyField, CourseKeyField, LearningContextKeyField, UsageKeyField -from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField from openedx.core.djangolib.markup import HTML @@ -86,7 +85,7 @@ class StudentModule(models.Model): """ objects = ChunkingManager() - id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name + id = models.BigAutoField(primary_key=True) # pylint: disable=invalid-name ## The XBlock/XModule type (e.g. "problem") module_type = models.CharField(max_length=32, db_index=True) diff --git a/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py b/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py index 12687dd7aa26..280d0f6d926a 100644 --- a/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py +++ b/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py @@ -2,7 +2,6 @@ from django.db import migrations, models import django.db.models.deletion -from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField from django.conf import settings def bump_pk_start(apps, schema_editor): @@ -45,7 +44,7 @@ class Migration(migrations.Migration): ('state', models.TextField(null=True, blank=True)), ('grade', models.FloatField(null=True, blank=True)), ('max_grade', models.FloatField(null=True, blank=True)), - ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)), + ('id', models.BigAutoField(primary_key=True)), ('student_module', models.ForeignKey(to='courseware.StudentModule', on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False)), ], options={ diff --git a/lms/djangoapps/coursewarehistoryextended/models.py b/lms/djangoapps/coursewarehistoryextended/models.py index 37de4a93ab58..592fe85133b2 100644 --- a/lms/djangoapps/coursewarehistoryextended/models.py +++ b/lms/djangoapps/coursewarehistoryextended/models.py @@ -19,7 +19,6 @@ from django.dispatch import receiver from lms.djangoapps.courseware.models import BaseStudentModuleHistory, StudentModule -from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField class StudentModuleHistoryExtended(BaseStudentModuleHistory): @@ -35,7 +34,7 @@ class Meta: get_latest_by = "created" index_together = ['student_module'] - id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name + id = models.BigAutoField(primary_key=True) # pylint: disable=invalid-name student_module = models.ForeignKey(StudentModule, db_index=True, db_constraint=False, on_delete=models.DO_NOTHING) diff --git a/lms/djangoapps/grades/migrations/0001_initial.py b/lms/djangoapps/grades/migrations/0001_initial.py index aac99bb659aa..ed6545c10a28 100644 --- a/lms/djangoapps/grades/migrations/0001_initial.py +++ b/lms/djangoapps/grades/migrations/0001_initial.py @@ -3,8 +3,6 @@ from django.db import migrations, models from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField -from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField - class Migration(migrations.Migration): @@ -17,7 +15,7 @@ class Migration(migrations.Migration): fields=[ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)), ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)), - ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)), + ('id', models.BigAutoField(primary_key=True)), ('user_id', models.IntegerField()), ('course_id', CourseKeyField(max_length=255)), ('usage_key', UsageKeyField(max_length=255)), diff --git a/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py b/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py index 182e4e6e9f87..63a15f0b7843 100644 --- a/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py +++ b/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py @@ -3,8 +3,6 @@ from django.db import migrations, models from opaque_keys.edx.django.models import CourseKeyField -from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField - class Migration(migrations.Migration): @@ -18,7 +16,7 @@ class Migration(migrations.Migration): fields=[ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)), ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)), - ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)), + ('id', models.BigAutoField(primary_key=True)), ('user_id', models.IntegerField(db_index=True)), ('course_id', CourseKeyField(max_length=255)), ('course_edited_timestamp', models.DateTimeField(verbose_name='Last content edit timestamp')), diff --git a/lms/djangoapps/grades/models.py b/lms/djangoapps/grades/models.py index a5608eb39afd..b846d0132704 100644 --- a/lms/djangoapps/grades/models.py +++ b/lms/djangoapps/grades/models.py @@ -26,7 +26,6 @@ from opaque_keys.edx.keys import CourseKey, UsageKey from simple_history.models import HistoricalRecords -from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField from lms.djangoapps.grades import events # lint-amnesty, pylint: disable=unused-import from openedx.core.lib.cache_utils import get_cache from lms.djangoapps.grades.signals.signals import ( @@ -321,7 +320,7 @@ class Meta: ] # primary key will need to be large for this table - id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name + id = models.BigAutoField(primary_key=True) # pylint: disable=invalid-name user_id = models.IntegerField(blank=False) course_id = CourseKeyField(blank=False, max_length=255) @@ -581,7 +580,7 @@ class Meta: ] # primary key will need to be large for this table - id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name + id = models.BigAutoField(primary_key=True) # pylint: disable=invalid-name user_id = models.IntegerField(blank=False, db_index=True) course_id = CourseKeyField(blank=False, max_length=255) From d3cc9e4e942423e2f3089553d3f7f3d87ae15052 Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Mon, 18 Nov 2024 14:19:51 +0500 Subject: [PATCH 09/13] fix: existing migrations. --- lms/djangoapps/courseware/migrations/0001_initial.py | 2 +- .../courseware/migrations/0008_move_idde_to_edx_when.py | 2 +- lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py | 2 +- lms/djangoapps/courseware/models.py | 2 +- lms/djangoapps/grades/migrations/0001_initial.py | 2 +- .../grades/migrations/0006_persistent_course_grades.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lms/djangoapps/courseware/migrations/0001_initial.py b/lms/djangoapps/courseware/migrations/0001_initial.py index 56fed9178c88..ff6d8a4bfd06 100644 --- a/lms/djangoapps/courseware/migrations/0001_initial.py +++ b/lms/djangoapps/courseware/migrations/0001_initial.py @@ -53,7 +53,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='StudentModule', fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.BigAutoField(primary_key=True, serialize=False, verbose_name='ID')), ('module_type', models.CharField(default='problem', max_length=32, db_index=True, choices=[('problem', 'problem'), ('video', 'video'), ('html', 'html'), ('course', 'course'), ('chapter', 'Section'), ('sequential', 'Subsection'), ('library_content', 'Library Content')])), ('module_state_key', UsageKeyField(max_length=255, db_column='module_id', db_index=True)), ('course_id', CourseKeyField(max_length=255, db_index=True)), diff --git a/lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py b/lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py index 632409025cfa..fab182163326 100644 --- a/lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py +++ b/lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py @@ -33,5 +33,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(move_overrides_to_edx_when) + migrations.RunPython(move_overrides_to_edx_when, reverse_code=migrations.RunPython.noop) ] diff --git a/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py b/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py index 8dae3c7be92f..d76c4c489b2b 100644 --- a/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py +++ b/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py @@ -51,6 +51,6 @@ class Migration(migrations.Migration): CsmBigInt( model_name='studentmodule', name='id', - field=models.BigAutoField(primary_key=True), + field=models.BigAutoField(primary_key=True, serialize=False, verbose_name='ID'), ) ] diff --git a/lms/djangoapps/courseware/models.py b/lms/djangoapps/courseware/models.py index 8ec4c164a202..78cf07f23d4d 100644 --- a/lms/djangoapps/courseware/models.py +++ b/lms/djangoapps/courseware/models.py @@ -85,7 +85,7 @@ class StudentModule(models.Model): """ objects = ChunkingManager() - id = models.BigAutoField(primary_key=True) # pylint: disable=invalid-name + id = models.BigAutoField(verbose_name='ID', primary_key=True) # pylint: disable=invalid-name ## The XBlock/XModule type (e.g. "problem") module_type = models.CharField(max_length=32, db_index=True) diff --git a/lms/djangoapps/grades/migrations/0001_initial.py b/lms/djangoapps/grades/migrations/0001_initial.py index ed6545c10a28..cedea6b3150b 100644 --- a/lms/djangoapps/grades/migrations/0001_initial.py +++ b/lms/djangoapps/grades/migrations/0001_initial.py @@ -15,7 +15,7 @@ class Migration(migrations.Migration): fields=[ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)), ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)), - ('id', models.BigAutoField(primary_key=True)), + ('id', models.BigAutoField(primary_key=True, serialize=False)), ('user_id', models.IntegerField()), ('course_id', CourseKeyField(max_length=255)), ('usage_key', UsageKeyField(max_length=255)), diff --git a/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py b/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py index 63a15f0b7843..650049803370 100644 --- a/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py +++ b/lms/djangoapps/grades/migrations/0006_persistent_course_grades.py @@ -16,7 +16,7 @@ class Migration(migrations.Migration): fields=[ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)), ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)), - ('id', models.BigAutoField(primary_key=True)), + ('id', models.BigAutoField(primary_key=True, serialize=False)), ('user_id', models.IntegerField(db_index=True)), ('course_id', CourseKeyField(max_length=255)), ('course_edited_timestamp', models.DateTimeField(verbose_name='Last content edit timestamp')), From 720c4e89a470dd27f0c8fc5caddaef8a3aa7baaa Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Mon, 18 Nov 2024 14:30:00 +0500 Subject: [PATCH 10/13] fix: existing migrations. --- lms/djangoapps/coursewarehistoryextended/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/coursewarehistoryextended/models.py b/lms/djangoapps/coursewarehistoryextended/models.py index 592fe85133b2..ddd098304586 100644 --- a/lms/djangoapps/coursewarehistoryextended/models.py +++ b/lms/djangoapps/coursewarehistoryextended/models.py @@ -34,7 +34,7 @@ class Meta: get_latest_by = "created" index_together = ['student_module'] - id = models.BigAutoField(primary_key=True) # pylint: disable=invalid-name + id = models.BigAutoField(primary_key=True, serialize=False) # pylint: disable=invalid-name student_module = models.ForeignKey(StudentModule, db_index=True, db_constraint=False, on_delete=models.DO_NOTHING) From 2dc00d3525797c5c47d32664f12e8fbd53c10a56 Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Mon, 18 Nov 2024 16:29:49 +0500 Subject: [PATCH 11/13] fix: mismatched migrations. update: regesterd CourseLocator with register_adapter --- .../migrations/0001_initial.py | 2 +- .../djangoapps/common_initialization/apps.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py b/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py index 280d0f6d926a..8f788aefa607 100644 --- a/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py +++ b/lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py @@ -44,7 +44,7 @@ class Migration(migrations.Migration): ('state', models.TextField(null=True, blank=True)), ('grade', models.FloatField(null=True, blank=True)), ('max_grade', models.FloatField(null=True, blank=True)), - ('id', models.BigAutoField(primary_key=True)), + ('id', models.BigAutoField(primary_key=True, serialize=False)), ('student_module', models.ForeignKey(to='courseware.StudentModule', on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False)), ], options={ diff --git a/openedx/core/djangoapps/common_initialization/apps.py b/openedx/core/djangoapps/common_initialization/apps.py index ccbe6d76d929..574a3f3b6154 100644 --- a/openedx/core/djangoapps/common_initialization/apps.py +++ b/openedx/core/djangoapps/common_initialization/apps.py @@ -4,6 +4,7 @@ from django.apps import AppConfig +from django.db import connection class CommonInitializationConfig(AppConfig): # lint-amnesty, pylint: disable=missing-class-docstring @@ -14,6 +15,7 @@ def ready(self): # Common settings validations for the LMS and CMS. from . import checks # lint-amnesty, pylint: disable=unused-import self._add_mimetypes() + self._add_required_adapters() @staticmethod def _add_mimetypes(): @@ -26,3 +28,17 @@ def _add_mimetypes(): mimetypes.add_type('application/x-font-opentype', '.otf') mimetypes.add_type('application/x-font-ttf', '.ttf') mimetypes.add_type('application/font-woff', '.woff') + + @staticmethod + def _add_required_adapters(): + if 'postgresql' in connection.vendor.lower(): + from opaque_keys.edx.locator import CourseLocator + from psycopg2.extensions import register_adapter, QuotedString + + + def adapt_course_locator(course_locator): + return QuotedString(course_locator._to_string()) + + + # Register the adapter + register_adapter(CourseLocator, adapt_course_locator) From bccd80a4f3ba6de4a1d471a2caacd4d730cba3b4 Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Mon, 18 Nov 2024 16:50:16 +0500 Subject: [PATCH 12/13] fix: quality issues --- .../core/djangoapps/common_initialization/apps.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openedx/core/djangoapps/common_initialization/apps.py b/openedx/core/djangoapps/common_initialization/apps.py index 574a3f3b6154..f6977c68897e 100644 --- a/openedx/core/djangoapps/common_initialization/apps.py +++ b/openedx/core/djangoapps/common_initialization/apps.py @@ -2,7 +2,6 @@ Common initialization app for the LMS and CMS """ - from django.apps import AppConfig from django.db import connection @@ -31,14 +30,15 @@ def _add_mimetypes(): @staticmethod def _add_required_adapters(): + """ + Register CourseLocator in psycopg2 extensions + :return: + """ if 'postgresql' in connection.vendor.lower(): from opaque_keys.edx.locator import CourseLocator - from psycopg2.extensions import register_adapter, QuotedString - - + from psycopg2.extensions import QuotedString, register_adapter def adapt_course_locator(course_locator): - return QuotedString(course_locator._to_string()) - + return QuotedString(course_locator._to_string()) # lint-amnesty, pylint: disable=protected-access # Register the adapter register_adapter(CourseLocator, adapt_course_locator) From 6d0f777ec80e8b02e99db67774ffad1eb9ca87db Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Fri, 22 Nov 2024 12:48:44 +0500 Subject: [PATCH 13/13] fix: quality issues --- openedx/core/djangoapps/common_initialization/apps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openedx/core/djangoapps/common_initialization/apps.py b/openedx/core/djangoapps/common_initialization/apps.py index f6977c68897e..36dc8375309b 100644 --- a/openedx/core/djangoapps/common_initialization/apps.py +++ b/openedx/core/djangoapps/common_initialization/apps.py @@ -37,7 +37,9 @@ def _add_required_adapters(): if 'postgresql' in connection.vendor.lower(): from opaque_keys.edx.locator import CourseLocator from psycopg2.extensions import QuotedString, register_adapter + def adapt_course_locator(course_locator): + return QuotedString(course_locator._to_string()) # lint-amnesty, pylint: disable=protected-access # Register the adapter