Updated caching strategy

backend/PennCourses/settings/base.py

@@ -186,6 +186,7 @@
 
 # Redis
 REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/1")
+REDIS_URL_BASE = REDIS_URL.rsplit("/", 1)[0]
 
 # Celery
 MESSAGE_BROKER_URL = REDIS_URL

backend/PennCourses/settings/development.py

@@ -16,5 +16,19 @@
 CACHES = {
     "default": {
         "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-    }
+    },
+    "blue": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_URL_BASE + "/2",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    },
+    "green": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_URL_BASE + "/3",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    },
 }

backend/PennCourses/settings/production.py

@@ -32,7 +32,22 @@
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
         },
-    }
+    },
+    "blue": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_URL_BASE + "/2",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    },
+    "green": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_URL_BASE + "/3",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    },
+
 }
 
 MOBILE_NOTIFICATION_SECRET = os.environ.get("MOBILE_NOTIFICATION_SECRET", "")

backend/courses/management/commands/registrarimport.py

@@ -1,7 +1,9 @@
 import logging
 
+from django.core.cache import caches
 from django.core.management.base import BaseCommand
 from tqdm import tqdm
+from review.management.commands.precompute_pcr_views import precompute_pcr_views
 
 from courses import registrar
 from courses.management.commands.loadstatus import set_all_status
@@ -44,6 +46,8 @@ def registrar_import(semester=None, query=""):
         # Make sure to load in summer course data as well
         # (cron job only does current semester, which is either fall or spring)
         registrar_import(semester=semester[:-1] + "B", query=query)
+
+    precompute_pcr_views()
 
 
 class Command(BaseCommand):

backend/review/management/commands/clearcache.py

@@ -2,33 +2,47 @@
 
 import redis
 from django.conf import settings
-from django.core.cache import cache
+from django.core.cache import caches
 from django.core.management import BaseCommand
 
-
-def clear_cache():
+def clear_cache(clear_pcr_cache=False, _cache_alias="default", _redis_delete_keys="*views.decorators.cache*"):
     # If we are not using redis as the cache backend, then we can delete everything from the cache.
+    del_count = -1
     if (
         settings.CACHES is None
-        or settings.CACHES.get("default").get("BACKEND") != "django_redis.cache.RedisCache"
+        or settings.CACHES.get(_cache_alias).get("BACKEND") != "django_redis.cache.RedisCache"
     ):
-        cache.clear()
-        return -1
+        caches[_cache_alias].clear()
+    else:
+        # If redis is the cache backend, then we need to be careful to only delete django cache entries,
+        # since celery also uses redis as a message broker backend.
+        redis_url = settings.CACHES.get(_cache_alias).get("LOCATION")
+        r = redis.Redis.from_url(redis_url)
+        del_count = 0
+        for key in r.scan_iter(_redis_delete_keys):
+            r.delete(key)
+            del_count += 1
 
-    # If redis is the cache backend, then we need to be careful to only delete django cache entries,
-    # since celery also uses redis as a message broker backend.
-    r = redis.Redis.from_url(settings.REDIS_URL)
-    del_count = 0
-    for key in r.scan_iter("*views.decorators.cache*"):
-        r.delete(key)
-        del_count += 1
+    if clear_pcr_cache:
+        # avoid circular import
+        if del_count == -1: # TODO: this hacky
+            del_count = 0
+        del_count += clear_cache(clear_pcr_cache=False, _cache_alias="green", _redis_delete_keys=f"*")
+        del_count += clear_cache(clear_pcr_cache=False, _cache_alias="blue", _redis_delete_keys=f"*")
     return del_count
 
 
 class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--clear_pcr_cache",
+            action="store_true",
+            help="If set, will clear the PCR cache as well.",
+        )
+
     def handle(self, *args, **options):
         root_logger = logging.getLogger("")
         root_logger.setLevel(logging.DEBUG)
-
-        del_count = clear_cache()
-        print(f"{del_count if del_count >=0 else 'all'} cache entries removed.")
+        del_count = clear_cache(clear_pcr_cache=options["clear_pcr_cache"])
+        print(f"{'At least ' if options['clear_pcr_cache'] else ''}"
+              f"{del_count if del_count >=0 else 'all'} cache entries removed.")

backend/review/management/commands/iscimport.py

@@ -270,7 +270,7 @@ def handle(self, *args, **kwargs):
         self.close_files(files)
         # invalidate cached views
         print("Invalidating cache...")
-        del_count = clear_cache()
+        del_count = clear_cache(clear_pcr_cache=True)
         print(f"{del_count if del_count >=0 else 'all'} cache entries removed.")
 
         gc.collect()

backend/review/management/commands/precompute_pcr_views.py

@@ -0,0 +1,55 @@
+
+from django.core.management.base import BaseCommand
+from tqdm import tqdm
+
+from review.views import manual_course_reviews 
+from courses.models import Topic
+from django.core.cache import caches
+from django.conf import settings
+from django.http import Http404
+import logging
+import redis
+
+def precompute_pcr_views(verbose=False):
+    blue_cache = caches["blue"]
+    green_cache = caches["green"]
+    green_cache.clear()
+
+    # Note: we only cache the most recent topic for each full_code (any other cache entries get overwritten)
+    for topic in tqdm(
+        Topic.objects
+        .all()
+        .select_related("most_recent")
+        .order_by("most_recent__semester"),
+        disable=not verbose
+    ):
+        course_id_list, course_code_list = zip(*topic.courses.values_list("id", "full_code"))
+        topic_id = ".".join([str(id) for id in sorted(course_id_list)])
+        if blue_cache.get(topic_id) is None:
+            try:
+                green_cache.set(
+                    topic_id,
+                    manual_course_reviews(course_code_list[0], topic.most_recent.semester),
+                )
+            except Http404:
+                logging.info(f"Topic returned 404 (topic_id {topic_id}, "
+                             f"course_code {course_code_list[0]}, semester {topic.most_recent.semester})")
+        else:
+            green_cache.set(topic_id, blue_cache.get(topic_id))
+
+        for course_code in course_code_list:
+            green_cache.set(course_code, topic_id)
+
+    blue_redis_url, blue_redis_db = settings.CACHES.get("blue").get("LOCATION").rsplit("/", 1)
+    green_redis_url, green_redis_db = settings.CACHES.get("green").get("LOCATION").rsplit("/", 1)
+    assert blue_redis_url == green_redis_url, "Expect blue and green to use the same redis instance! Aborting blue green swap."
+    r = redis.Redis.from_url(blue_redis_url)
+
+    # swap blue and green
+    r.swapdb(int(blue_redis_db), int(green_redis_db))
+
+class Command(BaseCommand):
+    help = "Precompute PCR views for all topics"
+
+    def handle(self, *args, **kwargs):
+        precompute_pcr_views(verbose=True)

backend/review/urls.py

@@ -18,7 +18,7 @@
 urlpatterns = [
     path(
         "course/<slug:course_code>",
-        cache_page(MONTH_IN_SECONDS)(course_reviews),
+        course_reviews,
         name="course-reviews",
     ),
     path(

backend/review/views.py

@@ -1,6 +1,7 @@
 from collections import Counter, defaultdict
 
 from dateutil.tz import gettz
+from django.core.cache import caches
 from django.db.models import F, Max, OuterRef, Q, Subquery, Value
 from django.http import Http404
 from django.shortcuts import get_object_or_404
@@ -129,13 +130,25 @@ def extra_metrics_section_filters_pcr(current_semester=None):
 )
 @permission_classes([IsAuthenticated])
 def course_reviews(request, course_code, semester=None):
+    request_semester = request.GET.get("semester")
+    cache = caches["blue"]
+    topic_id = cache.get(course_code)
+    if topic_id is None:
+        return Response(manual_course_reviews(course_code, request_semester, semester))
+    response = cache.get(topic_id)
+    if response is None:
+        return Response(manual_course_reviews(course_code, request_semester, semester))
+    return Response(response)
+
+
+def manual_course_reviews(course_code, request_semester, semester=None):
     """
     Get all reviews for the topic of a given course and other relevant information.
     Different aggregation views are provided, such as reviews spanning all semesters,
     only the most recent semester, and instructor-specific views.
     """
     try:
-        semester = request.GET.get("semester")
+        semester = request_semester
         course = (
             Course.objects.filter(
                 course_filters_pcr,
@@ -168,8 +181,8 @@ def course_reviews(request, course_code, semester=None):
             course_filters_pcr,
             full_code=course_code,
         )
-        .aggregate(max_semester=Max("semester"))
-        .get("max_semester")
+        .aggregate(max_semester=Max("semester", default="")) # default of "" handles case when no course is found
+        .get("max_semester", "")
         > course.semester
         if semester
         else False
@@ -247,23 +260,21 @@ def course_reviews(request, course_code, semester=None):
         section_filters_pcr,
         course__topic=topic,
     )
-
-    return Response(
-        {
-            "code": course["full_code"],
-            "last_offered_sem_if_superceded": last_offered_sem_if_superceded,
-            "name": course["title"],
-            "description": course["description"],
-            "aliases": aliases,
-            "historical_codes": historical_codes,
-            "latest_semester": course["semester"],
-            "num_sections": num_sections,
-            "num_sections_recent": num_sections_recent,
-            "instructors": instructors,
-            "registration_metrics": num_registration_metrics > 0,
-            **get_average_and_recent_dict_single(course),
-        }
-    )
+
+    return {
+        "code": course["full_code"],
+        "last_offered_sem_if_superceded": last_offered_sem_if_superceded,
+        "name": course["title"],
+        "description": course["description"],
+        "aliases": aliases,
+        "historical_codes": historical_codes,
+        "latest_semester": course["semester"],
+        "num_sections": num_sections,
+        "num_sections_recent": num_sections_recent,
+        "instructors": instructors,
+        "registration_metrics": num_registration_metrics > 0,
+        **get_average_and_recent_dict_single(course),
+    }
 
 
 @api_view(["GET"])