Updated caching strategy

backend/PennCourses/settings/base.py

@@ -238,3 +238,6 @@
 # The name of the schedule that is created/verified by Penn Mobile,
 # containing the user's active course registrations from Path.
 PATH_REGISTRATION_SCHEDULE_NAME = "Path Registration"
+
+# Manually Set Cache Prefix
+CACHE_PREFIX = "MANUAL_CACHE_"

backend/courses/management/commands/registrarimport.py

@@ -10,6 +10,7 @@
 from courses.models import Department, Section
 from courses.util import get_current_semester, upsert_course_from_opendata
 from review.management.commands.clearcache import clear_cache
+from review.management.commands.precompute_pcr_views import precompute_pcr_views
 
 
 def registrar_import(semester=None, query=""):
@@ -45,6 +46,8 @@ def registrar_import(semester=None, query=""):
         # (cron job only does current semester, which is either fall or spring)
         registrar_import(semester=semester[:-1] + "B", query=query)
 
+    precompute_pcr_views(verbose=True, is_new_data=False)
+
 
 class Command(BaseCommand):
     help = "Load in courses, sections and associated models from the Penn registrar and requirements data sources."  # noqa: E501

backend/review/management/commands/clearcache.py

@@ -5,6 +5,8 @@
 from django.core.cache import cache
 from django.core.management import BaseCommand
 
+from PennCourses.settings.base import CACHE_PREFIX
+
 
 def clear_cache():
     # If we are not using redis as the cache backend, then we can delete everything from the cache.
@@ -22,6 +24,9 @@ def clear_cache():
     for key in r.scan_iter("*views.decorators.cache*"):
         r.delete(key)
         del_count += 1
+    for key in r.scan_iter(f"*{CACHE_PREFIX}*"):
+        r.delete(key)
+        del_count += 1
     return del_count
 
 

backend/review/management/commands/iscimport.py

@@ -17,6 +17,7 @@
 )
 from review.import_utils.parse_sql import load_sql_dump
 from review.management.commands.clearcache import clear_cache
+from review.management.commands.precompute_pcr_views import precompute_pcr_views
 from review.models import Review
 
 
@@ -277,6 +278,7 @@ def handle(self, *args, **kwargs):
 
         print("Recomputing Section.has_reviews...")
         recompute_has_reviews()
+        precompute_pcr_views(verbose=True, is_new_data=True)
 
         print("Done.")
         return 0

backend/review/management/commands/precompute_pcr_views.py

@@ -0,0 +1,113 @@
+import logging
+
+from django.core.cache import cache
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from tqdm import tqdm
+
+from courses.models import Topic
+from PennCourses.settings.base import CACHE_PREFIX
+from review.models import CachedReviewResponse
+from review.views import manual_course_reviews
+
+
+def precompute_pcr_views(verbose=False, is_new_data=False):
+    if verbose:
+        print("Now precomputing PCR reviews.")
+
+    objs_to_insert = []
+    objs_to_update = []
+    cache_deletes = set()
+    valid_reviews_in_db = total_reviews = 0
+
+    with transaction.atomic():
+        # Mark all the topics as expired.
+        CachedReviewResponse.objects.all().update(expired=True)
+        cached_responses = CachedReviewResponse.objects.all()
+        topic_id_to_response_obj = {response.topic_id: response for response in cached_responses}
+
+        # Iterate through all topics.
+        for topic in tqdm(
+            Topic.objects.all().select_related("most_recent").order_by("most_recent__semester"),
+            disable=not verbose,
+        ):
+            # get topic id
+            course_id_list, course_code_list = zip(*topic.courses.values_list("id", "full_code"))
+            topic_id = ".".join([str(id) for id in sorted(course_id_list)])
+            total_reviews += 1
+
+            if topic_id in topic_id_to_response_obj:
+                # current topic id is already cached
+                valid_reviews_in_db += 1
+                response_obj = topic_id_to_response_obj[topic_id]
+                response_obj.expired = False
+
+                if is_new_data:
+                    cache_deletes.add(CACHE_PREFIX + topic_id)
+                    review_data = manual_course_reviews(
+                        topic.most_recent.full_code, topic.most_recent.semester
+                    )
+                    if not review_data:
+                        logging.info(
+                            f"Invalid review data for ("
+                            f"topic_id={topic_id},"
+                            f"course_code={course_code_list[0]},"
+                            f"semester={topic.most_recent.semester})"
+                        )
+                        continue
+                    response_obj.response = review_data
+                objs_to_update.append(response_obj)
+            else:
+                # current topic id is not cached
+                review_data = manual_course_reviews(
+                    topic.most_recent.full_code, topic.most_recent.semester
+                )
+                if not review_data:
+                    logging.info(
+                        f"Invalid review data for ("
+                        f"topic_id={topic_id},"
+                        f"course_code={course_code_list[0]},"
+                        f"semester={topic.most_recent.semester})"
+                    )
+                    continue
+
+                objs_to_insert.append(
+                    CachedReviewResponse(topic_id=topic_id, response=review_data, expired=False)
+                )
+                for course_code in course_code_list:
+                    curr_topic_id = cache.get(CACHE_PREFIX + course_code)
+                    if curr_topic_id:
+                        cache_deletes.add(CACHE_PREFIX + curr_topic_id)
+                    cache_deletes.add(CACHE_PREFIX + course_code)
+
+        if verbose:
+            print(
+                f"{total_reviews} course reviews covered, "
+                f"{valid_reviews_in_db} of which were already in the database. "
+                f"{len(objs_to_insert)} course reviews were created. "
+                f"{len(objs_to_update)} course reviews were updated."
+            )
+
+        # Bulk create / update objects.
+        if verbose:
+            print("Creating and updating objects.")
+        CachedReviewResponse.objects.bulk_create(objs_to_insert)
+        CachedReviewResponse.objects.bulk_update(objs_to_update, ["response", "expired"])
+
+        # Bulk delete objects.
+        if verbose:
+            print("Deleting expired objects.")
+        CachedReviewResponse.objects.filter(expired=True).delete()
+        cache.delete_many(cache_deletes)
+
+
+class Command(BaseCommand):
+    help = "Precompute PCR views for all topics"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--new_data", action="store_true", help="Include this flag to recalculate review data."
+        )
+
+    def handle(self, *args, **kwargs):
+        precompute_pcr_views(verbose=True, is_new_data=kwargs["new_data"])

backend/review/migrations/0006_cachedreviewresponse.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.2 on 2024-10-26 03:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("review", "0005_review_comments"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CachedReviewResponse",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("topic_id", models.CharField(db_index=True, max_length=1000, unique=True)),
+                ("response", models.JSONField()),
+                ("expired", models.BooleanField(default=True)),
+            ],
+        ),
+    ]

backend/review/models.py

@@ -94,6 +94,19 @@ def get_averages(topic_id, instructor_name=None, fields=None):
 ALL_FIELD_SLUGS = [x[2] for x in REVIEW_BIT_LABEL]
 
 
+class CachedReviewResponse(models.Model):
+    """
+    Represents a mapping from temp_topic_id (string-delimited list of sorted courses within a topic)
+    to a JSON object storing summarized course review data (all the data frontend uses to display
+    reviews).
+    """
+
+    # Using large string to account for topics with many course ids.
+    topic_id = models.CharField(max_length=1000, db_index=True, unique=True)
+    response = models.JSONField()
+    expired = models.BooleanField(default=True)
+
+
 class ReviewBit(models.Model):
     """
     A single key/value pair associated with a review. Fields are things like "course_quality",

backend/review/urls.py

@@ -18,7 +18,7 @@
 urlpatterns = [
     path(
         "course/<slug:course_code>",
-        cache_page(MONTH_IN_SECONDS)(course_reviews),
+        course_reviews,
         name="course-reviews",
     ),
     path(