Adding Celery retry config

common/celery.py

@@ -20,3 +20,7 @@
 # this should be automactically configured via ^^ config_from_object
 # but it isn't so here it's configured here
 app.conf.task_routes = settings.CELERY_ROUTES
+
+# this means the celery task only sends the acknowledgement (removes from task queue)
+# after it has completed
+# app.conf.task_ack_late = True # What's the impact on all of our tasks?

notifications/tasks.py

@@ -1,12 +1,20 @@
 import logging
 
 from celery import shared_task
+from django.conf import settings
 from django.db.transaction import atomic
 
 logger = logging.getLogger(__name__)
 
 
-@shared_task
+@shared_task(
+    default_retry_delay=settings.NOTIFICATIONS_DEFAULT_RETRY_DELAY,
+    max_retries=settings.NOTIFICATIONS_MAX_RETRIES,
+    retry_backoff=True,
+    retry_backoff_max=settings.NOTIFICATIONS_RETRY_BACKOFF_MAX,
+    retry_jitter=True,
+    autoretry_for=(Exception,),
+)
 @atomic
 def send_emails_task(notification_pk: int):
     """Task for emailing all users signed up to receive packaging updates and

publishing/tasks.py

@@ -11,7 +11,14 @@
 logger = logging.getLogger(__name__)
 
 
-@app.task
+@app.task(
+    default_retry_delay=settings.ENVELOPE_GENERATION_DEFAULT_RETRY_DELAY,
+    max_retries=settings.ENVELOPE_GENERATION_MAX_RETRIES,
+    retry_backoff=True,
+    retry_backoff_max=settings.ENVELOPE_GENERATION_RETRY_BACKOFF_MAX,
+    retry_jitter=True,
+    autoretry_for=(Exception,),
+)
 def create_xml_envelope_file(
     packaged_work_basket_id: int,
     notify_when_done: bool = True,

publishing/tests/test_tasks.py

@@ -38,7 +38,7 @@ def test_create_and_upload_envelope(
         create_xml_envelope_file.apply(
             (packaged_work_basket.pk, True),
         )
-        mock_save.assert_called_once()
+        mock_save.assert_called()
 
     assert expected_bucket in s3_bucket_names()
 

settings/common.py

@@ -437,16 +437,54 @@
 # Settings about retrying uploads if the api cannot be contacted.
 # Names correspond to celery settings for retrying tasks:
 #   https://docs.celeryq.dev/en/stable/userguide/tasks.html#automatic-retry-for-known-exceptions
+CELERY_MAX_RETRIES = int(
+    os.environ.get("CELERY_MAX_RETRIES", "3"),
+)
+CELERY_RETRY_BACKOFF_MAX = int(
+    os.environ.get("CELERY_RETRY_BACKOFF_MAX", "600"),
+)
+CELERY_DEFAULT_RETRY_DELAY = int(
+    os.environ.get("CELERY_DEFAULT_RETRY_DELAY", "8"),
+)
+
 CROWN_DEPENDENCIES_API_MAX_RETRIES = int(
-    os.environ.get("CROWN_DEPENDENCIES_API_MAX_RETRIES", "3"),
+    os.environ.get("CROWN_DEPENDENCIES_API_MAX_RETRIES", CELERY_MAX_RETRIES),
 )
 CROWN_DEPENDENCIES_API_RETRY_BACKOFF_MAX = int(
-    os.environ.get("CROWN_DEPENDENCIES_API_RETRY_BACKOFF_MAX", "600"),
+    os.environ.get(
+        "CROWN_DEPENDENCIES_API_RETRY_BACKOFF_MAX",
+        CELERY_RETRY_BACKOFF_MAX,
+    ),
 )
 CROWN_DEPENDENCIES_API_DEFAULT_RETRY_DELAY = int(
-    os.environ.get("CROWN_DEPENDENCIES_API_DEFAULT_RETRY_DELAY", "8"),
+    os.environ.get(
+        "CROWN_DEPENDENCIES_API_DEFAULT_RETRY_DELAY",
+        CELERY_DEFAULT_RETRY_DELAY,
+    ),
 )
 
+ENVELOPE_GENERATION_MAX_RETRIES = int(
+    os.environ.get("ENVELOPE_GENERATION_MAX_RETRIES", CELERY_MAX_RETRIES),
+)
+ENVELOPE_GENERATION_RETRY_BACKOFF_MAX = int(
+    os.environ.get("ENVELOPE_GENERATION_RETRY_BACKOFF_MAX", CELERY_RETRY_BACKOFF_MAX),
+)
+ENVELOPE_GENERATION_DEFAULT_RETRY_DELAY = int(
+    os.environ.get(
+        "ENVELOPE_GENERATION_DEFAULT_RETRY_DELAY",
+        CELERY_DEFAULT_RETRY_DELAY,
+    ),
+)
+
+NOTIFICATIONS_MAX_RETRIES = int(
+    os.environ.get("NOTIFICATIONS_MAX_RETRIES", CELERY_MAX_RETRIES),
+)
+NOTIFICATIONS_RETRY_BACKOFF_MAX = int(
+    os.environ.get("NOTIFICATIONS_RETRY_BACKOFF_MAX", CELERY_RETRY_BACKOFF_MAX),
+)
+NOTIFICATIONS_DEFAULT_RETRY_DELAY = int(
+    os.environ.get("NOTIFICATIONS_DEFAULT_RETRY_DELAY", CELERY_DEFAULT_RETRY_DELAY),
+)
 
 # SQLite AWS settings
 SQLITE_STORAGE_BUCKET_NAME = os.environ.get("SQLITE_STORAGE_BUCKET_NAME", "sqlite")