Updated caching strategy

backend/courses/management/commands/registrarimport.py

@@ -10,6 +10,7 @@
 from courses.models import Department, Section
 from courses.util import get_current_semester, upsert_course_from_opendata
 from review.management.commands.clearcache import clear_cache
+from review.management.commands.precompute_pcr_views import precompute_pcr_views
 
 
 def registrar_import(semester=None, query=""):
@@ -45,6 +46,8 @@ def registrar_import(semester=None, query=""):
         # (cron job only does current semester, which is either fall or spring)
         registrar_import(semester=semester[:-1] + "B", query=query)
 
+    precompute_pcr_views(True, False)
+
 
 class Command(BaseCommand):
     help = "Load in courses, sections and associated models from the Penn registrar and requirements data sources."  # noqa: E501

backend/courses/migrations/0064_alter_section_activity.py

@@ -0,0 +1,42 @@
+# Generated by Django 5.0.2 on 2024-03-22 23:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("courses", "0063_auto_20231212_1750"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="section",
+            name="activity",
+            field=models.CharField(
+                choices=[
+                    ("", "Undefined"),
+                    ("CLN", "Clinic"),
+                    ("CRT", "Clinical Rotation"),
+                    ("DAB", "Dissertation Abroad"),
+                    ("DIS", "Dissertation"),
+                    ("DPC", "Doctoral Program Exchange"),
+                    ("FLD", "Field Work"),
+                    ("HYB", "Hybrid"),
+                    ("IND", "Independent Study"),
+                    ("LAB", "Lab"),
+                    ("LEC", "Lecture"),
+                    ("MST", "Masters Thesis"),
+                    ("ONL", "Online"),
+                    ("PRC", "Practicum"),
+                    ("REC", "Recitation"),
+                    ("SEM", "Seminar"),
+                    ("SRT", "Senior Thesis"),
+                    ("STU", "Studio"),
+                ],
+                db_index=True,
+                help_text='The section activity, e.g. `LEC` for CIS-120-001 (2020A). Options and meanings: <table width=100%><tr><td>""</td><td>"Undefined"</td></tr><tr><td>"CLN"</td><td>"Clinic"</td></tr><tr><td>"CRT"</td><td>"Clinical Rotation"</td></tr><tr><td>"DAB"</td><td>"Dissertation Abroad"</td></tr><tr><td>"DIS"</td><td>"Dissertation"</td></tr><tr><td>"DPC"</td><td>"Doctoral Program Exchange"</td></tr><tr><td>"FLD"</td><td>"Field Work"</td></tr><tr><td>"HYB"</td><td>"Hybrid"</td></tr><tr><td>"IND"</td><td>"Independent Study"</td></tr><tr><td>"LAB"</td><td>"Lab"</td></tr><tr><td>"LEC"</td><td>"Lecture"</td></tr><tr><td>"MST"</td><td>"Masters Thesis"</td></tr><tr><td>"ONL"</td><td>"Online"</td></tr><tr><td>"PRC"</td><td>"Practicum"</td></tr><tr><td>"REC"</td><td>"Recitation"</td></tr><tr><td>"SEM"</td><td>"Seminar"</td></tr><tr><td>"SRT"</td><td>"Senior Thesis"</td></tr><tr><td>"STU"</td><td>"Studio"</td></tr></table>',
+                max_length=50,
+            ),
+        ),
+    ]

backend/review/management/commands/iscimport.py

@@ -17,6 +17,7 @@
 )
 from review.import_utils.parse_sql import load_sql_dump
 from review.management.commands.clearcache import clear_cache
+from review.management.commands.precompute_pcr_views import precompute_pcr_views
 from review.models import Review
 
 
@@ -277,6 +278,7 @@ def handle(self, *args, **kwargs):
 
         print("Recomputing Section.has_reviews...")
         recompute_has_reviews()
+        precompute_pcr_views(True, True)
 
         print("Done.")
         return 0

backend/review/management/commands/precompute_pcr_views.py

@@ -0,0 +1,101 @@
+import logging
+
+from django.core.cache import cache
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.http import Http404
+from tqdm import tqdm
+
+from courses.models import Topic
+from review.models import CachedReviewResponse
+from review.views import manual_course_reviews
+
+
+def precompute_pcr_views(verbose=False, is_new_data=False):
+    if verbose:
+        print("Now precomputing PCR reviews.")
+
+    CachedReviewResponse.objects.all().update(expired=True)
+    responses = CachedReviewResponse.objects.all()
+    topic_set = {response.topic_id: response for response in responses}
+
+    objs_to_insert = []
+    objs_to_update = []
+    cache_deletes = set()
+    has_count = total_count = 0
+
+    # Iterate through all topics.
+    with transaction.atomic():
+        for topic in tqdm(
+            Topic.objects.all().select_related("most_recent").order_by("most_recent__semester"),
+            disable=not verbose,
+        ):
+            course_id_list, course_code_list = zip(*topic.courses.values_list("id", "full_code"))
+            topic_id = ".".join([str(id) for id in sorted(course_id_list)])
+            total_count += 1
+
+            if topic_id in topic_set:
+                try:
+                    has_count += 1
+                    response_obj = topic_set[topic_id]
+                    if is_new_data:
+                        response_obj.response = manual_course_reviews(
+                            topic.most_recent.full_code, topic.most_recent.semester
+                        )
+                        cache_deletes.add(topic_id)
+                    response_obj.expired = False
+                    objs_to_update.append(response_obj)
+                except Http404:
+                    logging.info(
+                        f"Topic returned 404 (topic_id {topic_id}, "
+                        f"course_code {course_code_list[0]}, semester {topic.most_recent.semester})"
+                    )
+            else:
+                try:
+                    review_json = manual_course_reviews(
+                        topic.most_recent.full_code, topic.most_recent.semester
+                    )
+                    objs_to_insert.append(
+                        CachedReviewResponse(topic_id=topic_id, response=review_json, expired=False)
+                    )
+                    for course_code in course_code_list:
+                        curr_topic_id = cache.get(course_code)
+                        if curr_topic_id:
+                            cache_deletes.add(curr_topic_id)
+                        cache_deletes.add(course_code)
+                except Http404:
+                    logging.info(
+                        f"Topic returned 404 (topic_id {topic_id}, "
+                        f"course_code {course_code_list[0]}, semester {topic.most_recent.semester})"
+                    )
+
+        if verbose:
+            print(
+                f"{total_count} course reviews covered, {has_count} of which were already in the",
+                f" database. {len(objs_to_insert)} course reviews were created.",
+                f" {len(objs_to_update)} course reviews were updated.",
+            )
+
+        # Bulk create / update objects.
+        if verbose:
+            print("Creating and updating objects.")
+        CachedReviewResponse.objects.bulk_create(objs_to_insert)
+        CachedReviewResponse.objects.bulk_update(objs_to_update, ["response", "expired"])
+
+        # Bulk delete objects.
+        if verbose:
+            print("Deleting expired objects.")
+        CachedReviewResponse.objects.filter(expired=True).delete()
+        cache.delete_many(cache_deletes)
+
+
+class Command(BaseCommand):
+    help = "Precompute PCR views for all topics"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--new_data", action="store_true", help="Include this flag to recalculate review data."
+        )
+
+    def handle(self, *args, **kwargs):
+        precompute_pcr_views(verbose=True, is_new_data=kwargs["new_data"])

backend/review/migrations/0006_cachedreviewresponse.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.2 on 2024-03-22 23:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("review", "0005_review_comments"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CachedReviewResponse",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("topic_id", models.CharField(db_index=True, max_length=1000)),
+                ("response", models.JSONField()),
+                ("expired", models.BooleanField(default=True)),
+            ],
+        ),
+    ]

backend/review/models.py

@@ -94,6 +94,18 @@ def get_averages(topic_id, instructor_name=None, fields=None):
 ALL_FIELD_SLUGS = [x[2] for x in REVIEW_BIT_LABEL]
 
 
+class CachedReviewResponse(models.Model):
+    """
+    Represents a mapping from temp_topic_id (string-delimited list of sorted courses within a topic)
+    to a JSON object storing summarized course review data (all the data frontend uses to display
+    reviews).
+    """
+
+    topic_id = models.CharField(max_length=1000, db_index=True)
+    response = models.JSONField()
+    expired = models.BooleanField(default=True)
+
+
 class ReviewBit(models.Model):
     """
     A single key/value pair associated with a review. Fields are things like "course_quality",

backend/review/urls.py

@@ -18,7 +18,7 @@
 urlpatterns = [
     path(
         "course/<slug:course_code>",
-        cache_page(MONTH_IN_SECONDS)(course_reviews),
+        course_reviews,
         name="course-reviews",
     ),
     path(