PCR optimize instructor's course query

backend/review/annotations.py

@@ -45,6 +45,7 @@ def review_averages(
     prefix="",
     semester_aggregations=False,
     extra_metrics=True,
+    core_metrics=True,
 ):
     """
     Annotate the queryset with the average of all ReviewBits matching the given subfilters.
@@ -62,6 +63,7 @@ def review_averages(
     :param: extra_metrics: option to include extra metrics in PCR aggregations; final enrollment,
         percent of add/drop period open, average number of openings during add/drop,
         and percentage of sections filled in advance registration
+    :param: core_metrics: option to include core averages (e.g., average_difficulty). By default on.
     """
     from courses.models import Section, StatusUpdate
     from review.views import extra_metrics_section_filters_pcr
@@ -82,21 +84,25 @@ class FilledInAdvRegAvg(Subquery):
 
     queryset = queryset.annotate(
         **{
-            **{
-                (prefix + field): Subquery(
-                    ReviewBit.objects.filter(
-                        reviewbit_subfilters,
-                        field=field,
-                        review__responses__gt=0,
+            **(
+                {
+                    (prefix + field): Subquery(
+                        ReviewBit.objects.filter(
+                            reviewbit_subfilters,
+                            field=field,
+                            review__responses__gt=0,
+                        )
+                        .values("field")
+                        .order_by()
+                        .annotate(avg=Avg("average"))
+                        .values("avg")[:1],
+                        output_field=FloatField(),
                     )
-                    .values("field")
-                    .order_by()
-                    .annotate(avg=Avg("average"))
-                    .values("avg")[:1],
-                    output_field=FloatField(),
-                )
-                for field in fields
-            },
+                    for field in fields
+                }
+                if core_metrics
+                else dict()
+            ),
             **(
                 {
                     (prefix + "final_enrollment"): Subquery(
@@ -208,6 +214,7 @@ def annotate_with_matching_reviews(
     fields=None,
     prefix="",
     extra_metrics=True,
+    core_metrics=True,
 ):
     """
     Annotate each element the passed-in queryset with a subset of all review averages.
@@ -225,6 +232,7 @@ def annotate_with_matching_reviews(
     :param: extra_metrics: option to include extra metrics in PCR aggregations; final enrollment,
         percent of add/drop period open, average number of openings during add/drop,
         and percentage of sections filled in advance registration
+    :param: core_metrics: option to include core averages (e.g., average_difficulty). By default on.
     """
 
     from courses.models import Section  # avoid circular imports
@@ -255,11 +263,12 @@ def annotate_with_matching_reviews(
         prefix,
         semester_aggregations=True,
         extra_metrics=extra_metrics,
+        core_metrics=core_metrics
     )
 
 
 def annotate_average_and_recent(
-    qs, match_review_on, match_section_on, extra_metrics=True, fields=None
+    qs, match_review_on, match_section_on, extra_metrics=True, fields=None, core_metrics=True
 ):
     """
     Annotate queryset with both all reviews and recent reviews.
@@ -275,6 +284,8 @@ def annotate_average_and_recent(
         percent of add/drop period open, average number of openings during add/drop,
         and percentage of sections filled in advance registration
     :param: fields: option to specify the fields averaged by the query
+    :param: fields: option to specify the fields averaged by the query
+    :param: core_metrics: option to include core averages (e.g., average_difficulty). By default on.
     """
     qs = annotate_with_matching_reviews(
         qs,
@@ -284,6 +295,7 @@ def annotate_average_and_recent(
         prefix="average_",
         extra_metrics=extra_metrics,
         fields=fields,
+        core_metrics=core_metrics,
     )
     qs = annotate_with_matching_reviews(
         qs,
@@ -293,5 +305,6 @@ def annotate_average_and_recent(
         prefix="recent_",
         extra_metrics=extra_metrics,
         fields=fields,
+        core_metrics=core_metrics,
     )
     return qs

backend/review/views.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 
 from dateutil.tz import gettz
-from django.db.models import F, Max, OuterRef, Q, Subquery, Value
+from django.db.models import F, Max, OuterRef, Q, Subquery, Value, Avg, FloatField
 from django.http import Http404
 from django.shortcuts import get_object_or_404
 from rest_framework.decorators import api_view, permission_classes, schema
@@ -26,7 +26,7 @@
     instructor_for_course_reviews_response_schema,
     instructor_reviews_response_schema,
 )
-from review.models import ALL_FIELD_SLUGS, Review
+from review.models import ALL_FIELD_SLUGS, Review, ReviewBit
 from review.util import (
     aggregate_reviews,
     avg_and_recent_demand_plots,
@@ -91,7 +91,8 @@ def extra_metrics_section_filters_pcr(current_semester=None):
 
 
 course_filters_pcr_allow_xlist = ~Q(title="") | ~Q(description="") | Q(sections__has_reviews=True)
-course_filters_pcr = Q(primary_listing_id=F("id")) & course_filters_pcr_allow_xlist
+course_filters_pcr_require_primary_listing = Q(primary_listing_id=F("id"))
+course_filters_pcr = course_filters_pcr_require_primary_listing & course_filters_pcr_allow_xlist
 
 section_filters_pcr = Q(course__primary_listing_id=F("course_id")) & (
     Q(has_reviews=True)
@@ -372,6 +373,8 @@ def check_instructor_id(instructor_id):
     "difficulty",
 ]
 
+# TODO: remove before merging into prod
+USE_NEW_QUERIES = True
 
 @api_view(["GET"])
 @schema(
@@ -411,11 +414,49 @@ def instructor_reviews(request, instructor_id):
     )
     inst = get_single_dict_from_qs(instructor_qs)
 
+    courses = Course.objects.filter(
+        course_filters_pcr,
+        sections__instructors__id=instructor_id,
+    )
+
+    if USE_NEW_QUERIES:
+        courses = courses.annotate(**{
+            ("average_" + field): Subquery(
+                ReviewBit.objects.filter(
+                    review__responses__gt=0,
+                    review__section__course__topic=OuterRef("topic"),
+                    review__instructor_id=instructor_id,
+                    field=field
+                ).annotate(avg=Avg("average")).values("avg")[:1],
+                output_field=FloatField(),
+            )
+            for field in INSTRUCTOR_COURSE_REVIEW_FIELDS
+        })
+
+        courses = courses.annotate(max_semester=Subquery(
+            Review.objects.filter(  # Note: do not need review_filters_pcr since the outer course is a primary listing
+                section__course__id=OuterRef("id"),
+                instructor__id=instructor_id,
+                responses__gt=0,
+            ).annotate(max=Max("section__course__semester")).values("max")[:1]
+        )).values("max_semester")
+
+        # get the max semester for each course
+        courses = courses.annotate(**{
+            ("recent_" + field): Subquery(
+                ReviewBit.objects.filter(
+                    review__responses__gt=0,
+                    review__section__course__topic=OuterRef("topic"),
+                    review__instructor_id=instructor_id,
+                    field=field,
+                    review__section__course__semester=OuterRef("max_semester")
+                ).annotate(avg=Avg("average")).values("avg")[:1]
+            )
+            for field in INSTRUCTOR_COURSE_REVIEW_FIELDS
+        })
+
     courses = annotate_average_and_recent(
-        Course.objects.filter(
-            course_filters_pcr,
-            sections__instructors__id=instructor_id,
-        ).distinct(),
+        courses,
         match_review_on=Q(
             section__course__topic=OuterRef(OuterRef("topic")),
             instructor_id=instructor_id,
@@ -428,7 +469,10 @@ def instructor_reviews(request, instructor_id):
         & section_filters_pcr,
         extra_metrics=True,
         fields=INSTRUCTOR_COURSE_REVIEW_FIELDS,
-    ).annotate(
+        core_metrics=(not USE_NEW_QUERIES),
+    )
+
+    courses = courses.annotate(
         most_recent_full_code=F("topic__most_recent__full_code"),
     )
 
@@ -446,6 +490,7 @@ def instructor_reviews(request, instructor_id):
     courses_res = dict()
     max_sem = dict()
     for r in courses.values():
+        print(r)
         if not r["average_semester_count"]:
             continue
         full_code = r["most_recent_full_code"]