From c4ae425575ac1070801538f47a9325db3e22991d Mon Sep 17 00:00:00 2001 From: Aakash Singh Date: Sat, 7 Sep 2024 14:18:16 +0530 Subject: [PATCH 1/4] optimize events creation and improve indexing --- care/facility/api/viewsets/events.py | 6 ++-- care/facility/events/handler.py | 22 +++++++----- ...facility_pa_consult_7b22fe_idx_and_more.py | 35 +++++++++++++++++++ care/facility/models/events.py | 21 ++++++----- 4 files changed, 64 insertions(+), 20 deletions(-) create mode 100644 care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py diff --git a/care/facility/api/viewsets/events.py b/care/facility/api/viewsets/events.py index 32b81d3ab4..a0a7cb57cf 100644 --- a/care/facility/api/viewsets/events.py +++ b/care/facility/api/viewsets/events.py @@ -76,9 +76,9 @@ class PatientConsultationEventViewSet(ReadOnlyModelViewSet): def get_consultation_obj(self): return get_object_or_404( - get_consultation_queryset(self.request.user).filter( - external_id=self.kwargs["consultation_external_id"] - ) + get_consultation_queryset(self.request.user) + .filter(external_id=self.kwargs["consultation_external_id"]) + .only("id") ) def get_queryset(self): diff --git a/care/facility/events/handler.py b/care/facility/events/handler.py index 53c3ffc6ba..28f157d0c9 100644 --- a/care/facility/events/handler.py +++ b/care/facility/events/handler.py @@ -31,6 +31,7 @@ def create_consultation_event_entry( fields_to_store = fields_to_store & fields if fields_to_store else fields batch = [] + groups_to_mark_stale: list[int] = [] groups = EventType.objects.filter( model=object_instance.__class__.__name__, fields__len__gt=0, is_active=True ).values_list("id", "fields") @@ -44,14 +45,7 @@ def create_consultation_event_entry( if all(not v for v in value.values()): continue - PatientConsultationEvent.objects.select_for_update().filter( - consultation_id=consultation_id, - event_type=group_id, - is_latest=True, - object_model=object_instance.__class__.__name__, - object_id=object_instance.id, - taken_at__lt=taken_at, - ).update(is_latest=False) + groups_to_mark_stale.append(group_id) batch.append( PatientConsultationEvent( consultation_id=consultation_id, @@ -71,6 +65,18 @@ def create_consultation_event_entry( ) ) + old_events_filter = { + "consultation_id": consultation_id, + "event_type__in": groups_to_mark_stale, + "is_latest": True, + "object_model": object_instance.__class__.__name__, + "taken_at__lt": taken_at, + } + if change_type == ChangeType.UPDATED: + old_events_filter["object_id"] = object_instance.id + PatientConsultationEvent.objects.select_for_update().filter( + **old_events_filter + ).update(is_latest=False) PatientConsultationEvent.objects.bulk_create(batch) return len(batch) diff --git a/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py b/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py new file mode 100644 index 0000000000..2c490dbd2d --- /dev/null +++ b/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.15 on 2024-09-07 08:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("facility", "0454_remove_historicalpatientregistration_abha_number_and_more"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="patientconsultationevent", + name="facility_pa_consult_7b22fe_idx", + ), + migrations.AlterField( + model_name="patientconsultationevent", + name="is_latest", + field=models.BooleanField(db_index=True, default=True), + ), + migrations.AddIndex( + model_name="patientconsultationevent", + index=models.Index( + fields=[ + "consultation_id", + "is_latest", + "event_type_id", + "object_model", + "taken_at", + ], + name="facility_pa_consult_2153a6_idx", + ), + ), + ] diff --git a/care/facility/models/events.py b/care/facility/models/events.py index 9eafe47388..fd1628e733 100644 --- a/care/facility/models/events.py +++ b/care/facility/models/events.py @@ -61,7 +61,7 @@ class PatientConsultationEvent(models.Model): ) object_id = models.IntegerField(null=False, blank=False) event_type = models.ForeignKey(EventType, null=False, on_delete=models.PROTECT) - is_latest = models.BooleanField(default=True) + is_latest = models.BooleanField(default=True, db_index=True) meta = models.JSONField(default=dict, encoder=CustomJSONEncoder) value = models.JSONField(default=dict, encoder=CustomJSONEncoder) change_type = models.CharField( @@ -73,11 +73,14 @@ def __str__(self) -> str: class Meta: ordering = ["-created_date"] - indexes = [models.Index(fields=["consultation", "is_latest"])] - # constraints = [ - # models.UniqueConstraint( - # fields=["consultation", "event_type", "is_latest"], - # condition=models.Q(is_latest=True), - # name="unique_consultation_event_type_is_latest", - # ) - # ] + indexes = [ + models.Index( + fields=[ + "consultation_id", + "is_latest", + "event_type_id", + "object_model", + "taken_at", + ] + ), + ] From 4c3f3363e79e5cae71395967e36d00f442ec678b Mon Sep 17 00:00:00 2001 From: Aakash Singh Date: Sat, 7 Sep 2024 14:33:00 +0530 Subject: [PATCH 2/4] use partial index --- ...sultationevent_facility_pa_consult_7b22fe_idx_and_more.py | 5 +++-- care/facility/models/events.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py b/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py index 2c490dbd2d..4e6629e0f7 100644 --- a/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py +++ b/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.15 on 2024-09-07 08:46 +# Generated by Django 4.2.15 on 2024-09-07 09:02 from django.db import migrations, models @@ -22,6 +22,7 @@ class Migration(migrations.Migration): migrations.AddIndex( model_name="patientconsultationevent", index=models.Index( + condition=models.Q(("is_latest", True)), fields=[ "consultation_id", "is_latest", @@ -29,7 +30,7 @@ class Migration(migrations.Migration): "object_model", "taken_at", ], - name="facility_pa_consult_2153a6_idx", + name="consultation_events_latest_idx", ), ), ] diff --git a/care/facility/models/events.py b/care/facility/models/events.py index fd1628e733..dd4eb26812 100644 --- a/care/facility/models/events.py +++ b/care/facility/models/events.py @@ -81,6 +81,8 @@ class Meta: "event_type_id", "object_model", "taken_at", - ] + ], + condition=models.Q(is_latest=True), + name="consultation_events_latest_idx", ), ] From 43cb2068c5b482bb54fff0b68753adb78fdc5a9f Mon Sep 17 00:00:00 2001 From: Aakash Singh Date: Mon, 23 Sep 2024 01:01:25 +0530 Subject: [PATCH 3/4] rebase migrations --- ...tationevent_facility_pa_consult_7b22fe_idx_and_more.py} | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename care/facility/migrations/{0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py => 0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py} (82%) diff --git a/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py b/care/facility/migrations/0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py similarity index 82% rename from care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py rename to care/facility/migrations/0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py index 4e6629e0f7..c266c6f175 100644 --- a/care/facility/migrations/0455_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py +++ b/care/facility/migrations/0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py @@ -1,12 +1,13 @@ -# Generated by Django 4.2.15 on 2024-09-07 09:02 +# Generated by Django 5.1.1 on 2024-09-22 19:30 +from django.conf import settings from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ("facility", "0454_remove_historicalpatientregistration_abha_number_and_more"), + ("facility", "0463_patientnotes_reply_to"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ From e71d158da77e744b23276bf0fe406ef696ad17c5 Mon Sep 17 00:00:00 2001 From: Aakash Singh Date: Mon, 23 Sep 2024 10:43:40 +0530 Subject: [PATCH 4/4] rebase migrations --- care/facility/migrations/0465_merge_20240923_1043.py | 12 ++++++++++++ ...event_facility_pa_consult_7b22fe_idx_and_more.py} | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 care/facility/migrations/0465_merge_20240923_1043.py rename care/facility/migrations/{0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py => 0466_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py} (91%) diff --git a/care/facility/migrations/0465_merge_20240923_1043.py b/care/facility/migrations/0465_merge_20240923_1043.py new file mode 100644 index 0000000000..56a7872f2c --- /dev/null +++ b/care/facility/migrations/0465_merge_20240923_1043.py @@ -0,0 +1,12 @@ +# Generated by Django 5.1.1 on 2024-09-23 05:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("facility", "0464_alter_facilitycapacity_room_type_and_more"), + ("facility", "0464_rename_spo2_dailyround_archived_spo2"), + ] + + operations = [] diff --git a/care/facility/migrations/0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py b/care/facility/migrations/0466_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py similarity index 91% rename from care/facility/migrations/0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py rename to care/facility/migrations/0466_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py index c266c6f175..6f53113bae 100644 --- a/care/facility/migrations/0464_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py +++ b/care/facility/migrations/0466_remove_patientconsultationevent_facility_pa_consult_7b22fe_idx_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.1 on 2024-09-22 19:30 +# Generated by Django 5.1.1 on 2024-09-23 05:13 from django.conf import settings from django.db import migrations, models @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("facility", "0463_patientnotes_reply_to"), + ("facility", "0465_merge_20240923_1043"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ]