From f64d0639a22c047e3cda66fea2bbcc1932574601 Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Tue, 1 Oct 2024 08:24:19 +0530 Subject: [PATCH 1/6] Added materialise view for user invite --- commcare_connect/opportunity/helpers.py | 5 +- .../migrations/0059_userinvitesummary.py | 131 ++++++++++++++++++ commcare_connect/opportunity/models.py | 16 +++ commcare_connect/opportunity/tables.py | 6 +- commcare_connect/opportunity/tasks.py | 7 +- .../opportunity/tests/test_export.py | 5 + 6 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 commcare_connect/opportunity/migrations/0059_userinvitesummary.py diff --git a/commcare_connect/opportunity/helpers.py b/commcare_connect/opportunity/helpers.py index 7cd144b6..038d0728 100644 --- a/commcare_connect/opportunity/helpers.py +++ b/commcare_connect/opportunity/helpers.py @@ -9,11 +9,14 @@ OpportunityAccess, PaymentUnit, UserInvite, + UserInviteSummary, VisitValidationStatus, ) def get_annotated_opportunity_access(opportunity: Opportunity): + return UserInviteSummary.objects.filter(opportunity=opportunity) + learn_modules_count = opportunity.learn_app.learn_modules.count() access_objects = ( UserInvite.objects.filter(opportunity=opportunity) @@ -65,7 +68,7 @@ def get_annotated_opportunity_access(opportunity: Opportunity): ) .order_by("opportunity_access__user__name") ) - + print(str(access_objects.query)) return access_objects diff --git a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py new file mode 100644 index 00000000..221e2ba0 --- /dev/null +++ b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py @@ -0,0 +1,131 @@ +# Generated by Django 4.2.5 on 2024-09-30 14:21 +import django +from django.db import migrations, models +from django_celery_beat.models import CrontabSchedule, PeriodicTask + + +def create_refresh_materialized_view_task(apps, schema_editor): + schedule, _ = CrontabSchedule.objects.get_or_create( + minute="0", + hour="0", # At midnight + day_of_week="*", + day_of_month="*", + month_of_year="*", + ) + PeriodicTask.objects.update_or_create( + crontab=schedule, + name="refresh_materialized_view", + task="your_app_name.tasks.refresh_materialized_view", + ) + + +def delete_refresh_materialized_view_task(apps, schema_editor): + PeriodicTask.objects.filter( + name="refresh_materialized_view", + task="your_app_name.tasks.refresh_materialized_view", + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("opportunity", "0058_paymentinvoice_payment_invoice"), + ] + + """This migration exists because we created a materialized view for the `UserInviteSummary` model. + Django automatically generates migrations for all models present in the application. + Since `managed = False` and `db_table = "opportunity_userinvite_summary"`, Django will not create or modify this table in the database. + However, we still need this migration in the migration script to prevent Django from generating it again when `makemigrations` is run in the future. + This ensures that the model is recognized without altering the actual database structure.""" + + operations = [ + migrations.CreateModel( + name="UserInviteSummary", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "status", + models.CharField( + choices=[ + ("sms_delivered", "SMS Delivered"), + ("sms_not_delivered", "SMS Not Delivered"), + ("accepted", "Accepted"), + ("invited", "Invited"), + ("not_found", "ConnectID Not Found"), + ], + default="invited", + max_length=50, + ), + ), + ("last_visit_date", models.DateTimeField(blank=True, null=True)), + ("date_deliver_started", models.DateTimeField(blank=True, null=True)), + ("passed_assessment", models.IntegerField()), + ("completed_modules_count", models.IntegerField()), + ("job_claimed", models.DateTimeField(null=True)), + ("date_learn_completed", models.DateTimeField(null=True)), + ( + "opportunity", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="opportunity.opportunity"), + ), + ( + "opportunity_access", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="opportunity.opportunityaccess", + ), + ), + ], + options={ + "db_table": "opportunity_userinvite_summary", + "managed": False, + }, + ), + migrations.RunSQL( + sql=""" + CREATE MATERIALIZED VIEW opportunity_userinvite_summary AS + WITH total_learning_modules AS ( + SELECT + app_id, + COUNT(*) AS total_modules_count + FROM + opportunity_learnmodule + GROUP BY + app_id + ) + SELECT + userinvite.id AS id, + userinvite.status AS status, + userinvite.opportunity_access_id AS opportunity_access_id, + userinvite.opportunity_id AS opportunity_id, + MAX(uservisit.visit_date) FILTER (WHERE uservisit.opportunity_id = opp.id AND uservisit.status != 'trial') AS last_visit_date, + MIN(uservisit.visit_date) FILTER (WHERE uservisit.opportunity_id = opp.id) AS date_deliver_started, + SUM(CASE WHEN assessment.opportunity_id = opp.id AND assessment.passed = TRUE THEN 1 ELSE 0 END) AS passed_assessment, + COUNT(DISTINCT completedmodule.id) FILTER (WHERE completedmodule.opportunity_id = opp.id) AS completed_modules_count, + CASE WHEN claim.id IS NOT NULL THEN claim.date_claimed END AS job_claimed, + CASE + WHEN COUNT(completedmodule.id) = learning_module.total_modules_count THEN + MAX(completedmodule.date) FILTER (WHERE completedmodule.opportunity_id = opp.id) + END AS date_learn_completed + FROM + opportunity_userinvite AS userinvite + JOIN + opportunity_opportunityaccess AS access ON userinvite.opportunity_access_id = access.id + JOIN + users_user AS _user ON access.user_id = _user.id + LEFT JOIN + opportunity_uservisit AS uservisit ON _user.id = uservisit.user_id + LEFT JOIN + opportunity_assessment AS assessment ON _user.id = assessment.user_id + LEFT JOIN + opportunity_completedmodule AS completedmodule ON completedmodule.user_id = _user.id + LEFT JOIN + opportunity_opportunityclaim AS claim ON access.id = claim.opportunity_access_id + JOIN + opportunity_opportunity AS opp ON userinvite.opportunity_id = opp.id + JOIN + total_learning_modules AS learning_module ON learning_module.app_id = opp.learn_app_id + GROUP BY + userinvite.id, _user.id, claim.id, learning_module.total_modules_count, userinvite.opportunity_access_id; + """, + reverse_sql="DROP MATERIALIZED VIEW IF EXISTS opportunity_userinvite_summary;", + ), + ] diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py index 87d970c3..c4b16242 100644 --- a/commcare_connect/opportunity/models.py +++ b/commcare_connect/opportunity/models.py @@ -683,3 +683,19 @@ class CatchmentArea(models.Model): class Meta: unique_together = ("site_code", "opportunity") + + +class UserInviteSummary(models.Model): + opportunity_access = models.ForeignKey(OpportunityAccess, null=True, on_delete=models.DO_NOTHING) + opportunity = models.ForeignKey(Opportunity, null=True, on_delete=models.CASCADE) + status = models.CharField(max_length=50, choices=UserInviteStatus.choices, default=UserInviteStatus.invited) + last_visit_date = models.DateTimeField(null=True, blank=True) + date_deliver_started = models.DateTimeField(null=True, blank=True) + passed_assessment = models.IntegerField() # Rename to total_passed_assessments + completed_modules_count = models.IntegerField() # Rename to total_completed_modules_count + job_claimed = models.DateTimeField(null=True) # Rename to match SQL + date_learn_completed = models.DateTimeField(null=True) + + class Meta: + managed = False + db_table = "opportunity_userinvite_summary" diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py index a856650f..248ef0d8 100644 --- a/commcare_connect/opportunity/tables.py +++ b/commcare_connect/opportunity/tables.py @@ -10,8 +10,8 @@ Payment, PaymentInvoice, PaymentUnit, - UserInvite, UserInviteStatus, + UserInviteSummary, UserVisit, VisitValidationStatus, ) @@ -152,11 +152,11 @@ class UserStatusTable(OrgContextTable): completed_learning = AggregateColumn(verbose_name="Completed Learning", accessor="date_learn_completed") passed_assessment = BooleanAggregateColumn(verbose_name="Passed Assessment") started_delivery = AggregateColumn(verbose_name="Started Delivery", accessor="date_deliver_started") - last_visit_date = columns.Column(accessor="last_visit_date_d") + last_visit_date = columns.Column(accessor="last_visit_date") view_profile = columns.Column("View Profile", empty_values=(), footer=lambda table: f"Invited: {len(table.rows)}") class Meta: - model = UserInvite + model = UserInviteSummary fields = ("status",) sequence = ( "display_name", diff --git a/commcare_connect/opportunity/tasks.py b/commcare_connect/opportunity/tasks.py index b5450cc1..ec6d6fa5 100644 --- a/commcare_connect/opportunity/tasks.py +++ b/commcare_connect/opportunity/tasks.py @@ -6,7 +6,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import default_storage -from django.db import transaction +from django.db import connection, transaction from django.urls import reverse from django.utils.timezone import now from django.utils.translation import gettext @@ -339,3 +339,8 @@ def generate_catchment_area_export(opportunity_id: int, export_format: str): export_tmp_name = f"{now().isoformat()}_{opportunity.name}_catchment_area.{export_format}" save_export(dataset, export_tmp_name, export_format) return export_tmp_name + + +def refresh_materialized_view(): + with connection.cursor() as cursor: + cursor.execute("REFRESH MATERIALIZED VIEW opportunity_userinvite_summary;") diff --git a/commcare_connect/opportunity/tests/test_export.py b/commcare_connect/opportunity/tests/test_export.py index 32a94c55..bc4df061 100644 --- a/commcare_connect/opportunity/tests/test_export.py +++ b/commcare_connect/opportunity/tests/test_export.py @@ -13,6 +13,7 @@ ) from commcare_connect.opportunity.forms import DateRanges from commcare_connect.opportunity.models import Opportunity, UserInviteStatus, UserVisit +from commcare_connect.opportunity.tasks import refresh_materialized_view from commcare_connect.opportunity.tests.factories import ( AssessmentFactory, CatchmentAreaFactory, @@ -127,6 +128,7 @@ def test_export_user_status_table_no_data_only(opportunity: Opportunity): rows.append( (mobile_user.name, mobile_user.username, "Accepted", date.replace(tzinfo=None), "", False, "", "", "") ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") @@ -150,6 +152,7 @@ def test_export_user_status_table_learn_data_only(opportunity: Opportunity): rows.append( (mobile_user.name, mobile_user.username, "Accepted", date.replace(tzinfo=None), "", False, "", "", "") ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") @@ -191,6 +194,7 @@ def test_export_user_status_table_learn_assessment_data_only(opportunity: Opport "", ) ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") @@ -241,6 +245,7 @@ def test_export_user_status_table_data(opportunity: Opportunity): date.replace(tzinfo=None), ) ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") From d3a3d4b10c157a1a36a65ec20a287bf1ed6c920c Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Tue, 1 Oct 2024 08:31:32 +0530 Subject: [PATCH 2/6] Remove redundant function to streamline logic --- commcare_connect/opportunity/export.py | 8 ++-- commcare_connect/opportunity/helpers.py | 63 +------------------------ commcare_connect/opportunity/views.py | 5 +- 3 files changed, 6 insertions(+), 70 deletions(-) diff --git a/commcare_connect/opportunity/export.py b/commcare_connect/opportunity/export.py index 3e1cece8..b46f3aa8 100644 --- a/commcare_connect/opportunity/export.py +++ b/commcare_connect/opportunity/export.py @@ -6,15 +6,13 @@ from tablib import Dataset from commcare_connect.opportunity.forms import DateRanges -from commcare_connect.opportunity.helpers import ( - get_annotated_opportunity_access, - get_annotated_opportunity_access_deliver_status, -) +from commcare_connect.opportunity.helpers import get_annotated_opportunity_access_deliver_status from commcare_connect.opportunity.models import ( CatchmentArea, CompletedWork, Opportunity, OpportunityAccess, + UserInviteSummary, UserVisit, VisitValidationStatus, ) @@ -108,7 +106,7 @@ def export_empty_payment_table(opportunity: Opportunity) -> Dataset: def export_user_status_table(opportunity: Opportunity) -> Dataset: - access_objects = get_annotated_opportunity_access(opportunity) + access_objects = UserInviteSummary.objects.filter(opportunity=opportunity) table = UserStatusTable(access_objects, exclude=("date_popup", "view_profile")) return get_dataset(table, export_title="User status export") diff --git a/commcare_connect/opportunity/helpers.py b/commcare_connect/opportunity/helpers.py index 038d0728..b1b89dca 100644 --- a/commcare_connect/opportunity/helpers.py +++ b/commcare_connect/opportunity/helpers.py @@ -1,6 +1,6 @@ from collections import namedtuple -from django.db.models import Case, Count, F, Max, Min, Q, Sum, Value, When +from django.db.models import Count, F, Q, Value from commcare_connect.opportunity.models import ( CompletedWork, @@ -8,70 +8,9 @@ Opportunity, OpportunityAccess, PaymentUnit, - UserInvite, - UserInviteSummary, - VisitValidationStatus, ) -def get_annotated_opportunity_access(opportunity: Opportunity): - return UserInviteSummary.objects.filter(opportunity=opportunity) - - learn_modules_count = opportunity.learn_app.learn_modules.count() - access_objects = ( - UserInvite.objects.filter(opportunity=opportunity) - .select_related("opportunity_access", "opportunity_access__opportunityclaim", "opportunity_access__user") - .annotate( - last_visit_date_d=Max( - "opportunity_access__user__uservisit__visit_date", - filter=Q(opportunity_access__user__uservisit__opportunity=opportunity) - & ~Q(opportunity_access__user__uservisit__status=VisitValidationStatus.trial), - ), - date_deliver_started=Min( - "opportunity_access__user__uservisit__visit_date", - filter=Q(opportunity_access__user__uservisit__opportunity=opportunity), - ), - passed_assessment=Sum( - Case( - When( - Q( - opportunity_access__user__assessments__opportunity=opportunity, - opportunity_access__user__assessments__passed=True, - ), - then=1, - ), - default=0, - ) - ), - completed_modules_count=Count( - "opportunity_access__user__completed_modules", - filter=Q(opportunity_access__user__completed_modules__opportunity=opportunity), - distinct=True, - ), - job_claimed=Case( - When( - Q(opportunity_access__opportunityclaim__isnull=False), - then="opportunity_access__opportunityclaim__date_claimed", - ) - ), - ) - .annotate( - date_learn_completed=Case( - When( - Q(completed_modules_count=learn_modules_count), - then=Max( - "opportunity_access__user__completed_modules__date", - filter=Q(opportunity_access__user__completed_modules__opportunity=opportunity), - ), - ) - ) - ) - .order_by("opportunity_access__user__name") - ) - print(str(access_objects.query)) - return access_objects - - def get_annotated_opportunity_access_deliver_status(opportunity: Opportunity): access_objects = [] for payment_unit in opportunity.paymentunit_set.all(): diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py index 9b6c00aa..fbd9d910 100644 --- a/commcare_connect/opportunity/views.py +++ b/commcare_connect/opportunity/views.py @@ -44,7 +44,6 @@ VisitExportForm, ) from commcare_connect.opportunity.helpers import ( - get_annotated_opportunity_access, get_annotated_opportunity_access_deliver_status, get_payment_report_data, ) @@ -64,6 +63,7 @@ Payment, PaymentInvoice, PaymentUnit, + UserInviteSummary, UserVisit, VisitValidationStatus, ) @@ -478,8 +478,7 @@ def get_queryset(self): opportunity_id = self.kwargs["pk"] org_slug = self.kwargs["org_slug"] opportunity = get_opportunity_or_404(org_slug=org_slug, pk=opportunity_id) - access_objects = get_annotated_opportunity_access(opportunity) - return access_objects + return UserInviteSummary.objects.filter(opportunity=opportunity) @org_member_required From 12b14489f761af3c434897de0ebc12134c227b9b Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Tue, 1 Oct 2024 08:34:35 +0530 Subject: [PATCH 3/6] Removed redundant comments --- commcare_connect/opportunity/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py index c4b16242..431dc845 100644 --- a/commcare_connect/opportunity/models.py +++ b/commcare_connect/opportunity/models.py @@ -691,9 +691,9 @@ class UserInviteSummary(models.Model): status = models.CharField(max_length=50, choices=UserInviteStatus.choices, default=UserInviteStatus.invited) last_visit_date = models.DateTimeField(null=True, blank=True) date_deliver_started = models.DateTimeField(null=True, blank=True) - passed_assessment = models.IntegerField() # Rename to total_passed_assessments - completed_modules_count = models.IntegerField() # Rename to total_completed_modules_count - job_claimed = models.DateTimeField(null=True) # Rename to match SQL + passed_assessment = models.IntegerField() + completed_modules_count = models.IntegerField() + job_claimed = models.DateTimeField(null=True) date_learn_completed = models.DateTimeField(null=True) class Meta: From a4cb2cf505c9c660faaafdc68e022e8f87e0f206 Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Tue, 1 Oct 2024 08:36:53 +0530 Subject: [PATCH 4/6] updated default value for integer column --- .../opportunity/migrations/0059_userinvitesummary.py | 4 ++-- commcare_connect/opportunity/models.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py index 221e2ba0..7251fe70 100644 --- a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py +++ b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py @@ -58,8 +58,8 @@ class Migration(migrations.Migration): ), ("last_visit_date", models.DateTimeField(blank=True, null=True)), ("date_deliver_started", models.DateTimeField(blank=True, null=True)), - ("passed_assessment", models.IntegerField()), - ("completed_modules_count", models.IntegerField()), + ("passed_assessment", models.IntegerField(default=0)), + ("completed_modules_count", models.IntegerField(default=0)), ("job_claimed", models.DateTimeField(null=True)), ("date_learn_completed", models.DateTimeField(null=True)), ( diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py index 431dc845..755e5a1b 100644 --- a/commcare_connect/opportunity/models.py +++ b/commcare_connect/opportunity/models.py @@ -691,8 +691,8 @@ class UserInviteSummary(models.Model): status = models.CharField(max_length=50, choices=UserInviteStatus.choices, default=UserInviteStatus.invited) last_visit_date = models.DateTimeField(null=True, blank=True) date_deliver_started = models.DateTimeField(null=True, blank=True) - passed_assessment = models.IntegerField() - completed_modules_count = models.IntegerField() + passed_assessment = models.IntegerField(default=0) + completed_modules_count = models.IntegerField(default=0) job_claimed = models.DateTimeField(null=True) date_learn_completed = models.DateTimeField(null=True) From afc0c6eb2aafbaad0dc5e097bf5e25ff4328e1ed Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Tue, 1 Oct 2024 09:03:51 +0530 Subject: [PATCH 5/6] Schedule the celery task --- .../opportunity/migrations/0059_userinvitesummary.py | 5 +++-- commcare_connect/opportunity/tasks.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py index 7251fe70..ceaf914d 100644 --- a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py +++ b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py @@ -15,14 +15,14 @@ def create_refresh_materialized_view_task(apps, schema_editor): PeriodicTask.objects.update_or_create( crontab=schedule, name="refresh_materialized_view", - task="your_app_name.tasks.refresh_materialized_view", + task="commcare_connect.opportunity.tasks.refresh_materialized_view", ) def delete_refresh_materialized_view_task(apps, schema_editor): PeriodicTask.objects.filter( name="refresh_materialized_view", - task="your_app_name.tasks.refresh_materialized_view", + task="commcare_connect.opportunity.tasks.refresh_materialized_view", ).delete() @@ -128,4 +128,5 @@ class Migration(migrations.Migration): """, reverse_sql="DROP MATERIALIZED VIEW IF EXISTS opportunity_userinvite_summary;", ), + migrations.RunPython(create_refresh_materialized_view_task, delete_refresh_materialized_view_task), ] diff --git a/commcare_connect/opportunity/tasks.py b/commcare_connect/opportunity/tasks.py index ec6d6fa5..21055b26 100644 --- a/commcare_connect/opportunity/tasks.py +++ b/commcare_connect/opportunity/tasks.py @@ -341,6 +341,7 @@ def generate_catchment_area_export(opportunity_id: int, export_format: str): return export_tmp_name +@celery_app.task() def refresh_materialized_view(): with connection.cursor() as cursor: cursor.execute("REFRESH MATERIALIZED VIEW opportunity_userinvite_summary;") From 2ea6a1e5c8d6d970b9c7dd3f96a0ac1fce9f21ad Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Tue, 1 Oct 2024 11:42:31 +0530 Subject: [PATCH 6/6] added view for delivery summary --- commcare_connect/opportunity/export.py | 4 +- commcare_connect/opportunity/helpers.py | 71 +--------------- .../migrations/0059_userinvitesummary.py | 85 ++++++++++++++++++- commcare_connect/opportunity/models.py | 17 ++++ commcare_connect/opportunity/tasks.py | 6 +- .../opportunity/tests/test_helpers.py | 14 +-- commcare_connect/opportunity/views.py | 9 +- 7 files changed, 121 insertions(+), 85 deletions(-) diff --git a/commcare_connect/opportunity/export.py b/commcare_connect/opportunity/export.py index b46f3aa8..cea1190d 100644 --- a/commcare_connect/opportunity/export.py +++ b/commcare_connect/opportunity/export.py @@ -6,12 +6,12 @@ from tablib import Dataset from commcare_connect.opportunity.forms import DateRanges -from commcare_connect.opportunity.helpers import get_annotated_opportunity_access_deliver_status from commcare_connect.opportunity.models import ( CatchmentArea, CompletedWork, Opportunity, OpportunityAccess, + OpportunityDeliverySummary, UserInviteSummary, UserVisit, VisitValidationStatus, @@ -112,7 +112,7 @@ def export_user_status_table(opportunity: Opportunity) -> Dataset: def export_deliver_status_table(opportunity: Opportunity) -> Dataset: - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) table = DeliverStatusTable(access_objects, exclude=("details", "date_popup")) return get_dataset(table, export_title="Deliver Status export") diff --git a/commcare_connect/opportunity/helpers.py b/commcare_connect/opportunity/helpers.py index b1b89dca..0d36544c 100644 --- a/commcare_connect/opportunity/helpers.py +++ b/commcare_connect/opportunity/helpers.py @@ -1,75 +1,6 @@ from collections import namedtuple -from django.db.models import Count, F, Q, Value - -from commcare_connect.opportunity.models import ( - CompletedWork, - CompletedWorkStatus, - Opportunity, - OpportunityAccess, - PaymentUnit, -) - - -def get_annotated_opportunity_access_deliver_status(opportunity: Opportunity): - access_objects = [] - for payment_unit in opportunity.paymentunit_set.all(): - access_objects += ( - OpportunityAccess.objects.filter(opportunity=opportunity) - .select_related("user") - .annotate( - payment_unit=Value(payment_unit.name), - pending=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.pending, - ), - distinct=True, - ), - approved=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.approved, - ), - distinct=True, - ), - rejected=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.rejected, - ), - distinct=True, - ), - over_limit=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.over_limit, - ), - distinct=True, - ), - incomplete=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.incomplete, - ), - distinct=True, - ), - completed=F("approved") + F("rejected") + F("pending") + F("over_limit"), - ) - .order_by("user__name") - ) - access_objects.sort(key=lambda a: a.user.name) - return access_objects +from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, Opportunity, PaymentUnit def get_payment_report_data(opportunity: Opportunity): diff --git a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py index ceaf914d..dc6c1270 100644 --- a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py +++ b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py @@ -33,7 +33,7 @@ class Migration(migrations.Migration): """This migration exists because we created a materialized view for the `UserInviteSummary` model. Django automatically generates migrations for all models present in the application. - Since `managed = False` and `db_table = "opportunity_userinvite_summary"`, Django will not create or modify this table in the database. + Since `managed = False` and `db_table is provided Django will not create or modify this table in the database. However, we still need this migration in the migration script to prevent Django from generating it again when `makemigrations` is run in the future. This ensures that the model is recognized without altering the actual database structure.""" @@ -128,5 +128,88 @@ class Migration(migrations.Migration): """, reverse_sql="DROP MATERIALIZED VIEW IF EXISTS opportunity_userinvite_summary;", ), + migrations.CreateModel( + name="OpportunityDeliverySummary", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("approved", models.IntegerField(default=0)), + ("pending", models.IntegerField(default=0)), + ("rejected", models.IntegerField(default=0)), + ("over_limit", models.IntegerField(default=0)), + ("incomplete", models.IntegerField(default=0)), + ("completed", models.IntegerField(default=0)), + ("payment_unit", models.CharField(max_length=255)), + ( + "opportunity", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="opportunity.opportunity"), + ), + ( + "user", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="users.user"), + ), + ( + "opportunity_access", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="opportunity.opportunityaccess", + ), + ), + ], + options={ + "db_table": "opportunity_delivery_summary", + "managed": False, + }, + ), + migrations.RunSQL( + sql=""" + CREATE MATERIALIZED VIEW opportunity_delivery_summary AS + SELECT + access.id AS id, + access.id AS opportunity_access_id, + access.opportunity_id AS opportunity_id, + access.user_id AS user_id, + payment_unit.name AS payment_unit, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'pending' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS pending, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'approved' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS approved, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'rejected' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS rejected, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'over_limit' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS over_limit, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'incomplete' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS incomplete, + + COALESCE( + COUNT(DISTINCT CASE WHEN completed_work.status IN ('approved', 'rejected', 'pending', 'over_limit') + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END), + 0 + ) AS completed + FROM + opportunity_opportunityaccess AS access + LEFT JOIN + opportunity_completedwork AS completed_work ON access.id = completed_work.opportunity_access_id + LEFT JOIN + opportunity_paymentunit AS payment_unit ON completed_work.payment_unit_id = payment_unit.id + INNER JOIN + users_user AS _user ON access.user_id = _user.id + GROUP BY + access.id, + _user.id, + payment_unit.name; + """, + reverse_sql="DROP MATERIALIZED VIEW IF EXISTS opportunity_delivery_summary;", + ), migrations.RunPython(create_refresh_materialized_view_task, delete_refresh_materialized_view_task), ] diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py index 755e5a1b..f976f689 100644 --- a/commcare_connect/opportunity/models.py +++ b/commcare_connect/opportunity/models.py @@ -699,3 +699,20 @@ class UserInviteSummary(models.Model): class Meta: managed = False db_table = "opportunity_userinvite_summary" + + +class OpportunityDeliverySummary(models.Model): + opportunity_access = models.ForeignKey(OpportunityAccess, null=True, on_delete=models.DO_NOTHING) + opportunity = models.ForeignKey(Opportunity, null=True, on_delete=models.CASCADE) + user = models.ForeignKey(User, null=True, on_delete=models.CASCADE) + approved = models.IntegerField(default=0) + pending = models.IntegerField(default=0) + rejected = models.IntegerField(default=0) + over_limit = models.IntegerField(default=0) + incomplete = models.IntegerField(default=0) + completed = models.IntegerField(default=0) + payment_unit = models.CharField(max_length=255) + + class Meta: + managed = False + db_table = "opportunity_delivery_summary" diff --git a/commcare_connect/opportunity/tasks.py b/commcare_connect/opportunity/tasks.py index 21055b26..daf4a09b 100644 --- a/commcare_connect/opportunity/tasks.py +++ b/commcare_connect/opportunity/tasks.py @@ -344,4 +344,8 @@ def generate_catchment_area_export(opportunity_id: int, export_format: str): @celery_app.task() def refresh_materialized_view(): with connection.cursor() as cursor: - cursor.execute("REFRESH MATERIALIZED VIEW opportunity_userinvite_summary;") + cursor.execute( + """REFRESH MATERIALIZED VIEW opportunity_userinvite_summary; + REFRESH MATERIALIZED VIEW opportunity_delivery_summary; + """ + ) diff --git a/commcare_connect/opportunity/tests/test_helpers.py b/commcare_connect/opportunity/tests/test_helpers.py index c0528364..c667c69f 100644 --- a/commcare_connect/opportunity/tests/test_helpers.py +++ b/commcare_connect/opportunity/tests/test_helpers.py @@ -1,7 +1,7 @@ import pytest -from commcare_connect.opportunity.helpers import get_annotated_opportunity_access_deliver_status -from commcare_connect.opportunity.models import Opportunity +from commcare_connect.opportunity.models import Opportunity, OpportunityDeliverySummary +from commcare_connect.opportunity.tasks import refresh_materialized_view from commcare_connect.opportunity.tests.factories import ( CompletedWorkFactory, OpportunityAccessFactory, @@ -15,7 +15,8 @@ def test_deliver_status_query_no_visits(opportunity: Opportunity): mobile_users = MobileUserFactory.create_batch(5) for mobile_user in mobile_users: OpportunityAccessFactory(opportunity=opportunity, user=mobile_user, accepted=True) - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + refresh_materialized_view() + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) usernames = {user.username for user in mobile_users} for access in access_objects: @@ -41,7 +42,8 @@ def test_deliver_status_query(opportunity: Opportunity): count_by_status["completed"] = len(completed_works) - count_by_status["incomplete"] completed_work_counts[(mobile_user.username, pu.name)] = count_by_status - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + refresh_materialized_view() + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) for access in access_objects: username = access.user.username assert (username, access.payment_unit) in completed_work_counts @@ -61,7 +63,9 @@ def test_deliver_status_query_visits_another_opportunity(opportunity: Opportunit for mobile_user in mobile_users: OpportunityAccessFactory(opportunity=opportunity, user=mobile_user, accepted=True) CompletedWorkFactory.create_batch(5) - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + + refresh_materialized_view() + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) usernames = {user.username for user in mobile_users} for access in access_objects: assert access.user.username in usernames diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py index fbd9d910..f5b8e6dd 100644 --- a/commcare_connect/opportunity/views.py +++ b/commcare_connect/opportunity/views.py @@ -43,10 +43,7 @@ SendMessageMobileUsersForm, VisitExportForm, ) -from commcare_connect.opportunity.helpers import ( - get_annotated_opportunity_access_deliver_status, - get_payment_report_data, -) +from commcare_connect.opportunity.helpers import get_payment_report_data from commcare_connect.opportunity.models import ( BlobMeta, CatchmentArea, @@ -59,6 +56,7 @@ OpportunityAccess, OpportunityClaim, OpportunityClaimLimit, + OpportunityDeliverySummary, OpportunityVerificationFlags, Payment, PaymentInvoice, @@ -652,8 +650,7 @@ def get_queryset(self): opportunity_id = self.kwargs["pk"] org_slug = self.kwargs["org_slug"] opportunity = get_opportunity_or_404(pk=opportunity_id, org_slug=org_slug) - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) - return access_objects + return OpportunityDeliverySummary.objects.filter(opportunity=opportunity) @org_member_required