Merge pull request #5316 from akvo/send-link-report-in-email

Send link report in email

akvo/rsr/tests/views/py_reports/test_utils.py

@@ -0,0 +1,89 @@
+import io
+import os
+import binascii
+from datetime import timedelta
+from typing import cast
+from django.core import mail
+from django.core.files.storage import Storage, default_storage
+from django.test import override_settings
+from django.utils import timezone
+from django_q.models import Task
+from akvo.rsr.tests.base import BaseTestCase
+
+from akvo.rsr.views.py_reports.utils import REPORTS_STORAGE_BASE_DIR, cleanup_expired_reports, notify_dev_on_failed_task, notify_user_on_failed_report, save_report_file
+
+default_storage = cast(Storage, default_storage)
+
+
+class StorageTestCase(BaseTestCase):
+ def tearDown(self):
+ _, files = default_storage.listdir(REPORTS_STORAGE_BASE_DIR)
+ for f in files:
+ default_storage.delete(os.path.join(REPORTS_STORAGE_BASE_DIR, f))
+ super().tearDown()
+
+ def setUp(self):
+ super().setUp()
+ buffer = io.BytesIO(b'test')
+ self.url = save_report_file(REPORTS_STORAGE_BASE_DIR, 'test.txt', buffer.getvalue())
+ self.file_path = os.path.join(REPORTS_STORAGE_BASE_DIR, 'test.txt')
+
+ def test_save_report_file(self):
+ self.assertTrue(default_storage.exists(self.file_path))
+ self.assertEqual(default_storage.url(self.file_path), self.url)
+
+ def test_cleanup_expired_reports_no_files_deleted(self):
+ now = timezone.now()
+ t23 = now + timedelta(hours=23, minutes=58)
+ cleanup_expired_reports(t23)
+ self.assertTrue(default_storage.exists(self.file_path))
+
+ def test_cleanup_expired_reports_deletes_file(self):
+ now = timezone.now()
+ t24 = now + timedelta(hours=24, minutes=2)
+ cleanup_expired_reports(t24)
+ self.assertFalse(default_storage.exists(self.file_path))
+
+
+@override_settings(REPORT_ERROR_RECIPIENTS=['[email protected]'])
+class NotifyErrorTestCase(BaseTestCase):
+ def setUp(self):
+ super().setUp()
+ self.create_user('[email protected]')
+ self.failed_task = Task.objects.create(
+ id=self._generate_id(),
+ name='test',
+ args=({'report_label': 'test report'}, '[email protected]'),
+ result='error',
+ success=False,
+ started=timezone.now(),
+ stopped=timezone.now()
+ )
+ self.success_task = Task.objects.create(
+ id=self._generate_id(),
+ name='test',
+ args=({'report_label': 'test report'}, '[email protected]'),
+ success=True,
+ started=timezone.now(),
+ stopped=timezone.now()
+ )
+
+ def _generate_id(self):
+ return binascii.b2a_hex(os.urandom(16)).decode('utf-8')
+
+ def test_notify_dev(self):
+ notify_dev_on_failed_task(self.failed_task)
+ self.assertEqual(len(mail.outbox), 1)
+ self.assertIn('[email protected]', mail.outbox[0].to)
+
+ def test_notify_user(self):
+ notify_user_on_failed_report(self.failed_task)
+ self.assertEqual(len(mail.outbox), 2)
+ self.assertEqual(
+ [it for mail in mail.outbox for it in mail.to],
+ ['[email protected]', '[email protected]']
+ )
+
+ def test_ignore_success(self):
+ notify_user_on_failed_report(self.success_task)
+ self.assertEqual(len(mail.outbox), 0)

akvo/rsr/views/py_reports/results_indicators_excel_report.py

@@ -27,12 +27,20 @@
 @login_required
 def add_email_report_job(request, org_id):
  organisation = get_object_or_404(Organisation, pk=org_id)
+ report_label = f'Results and Indicators Export for {organisation.name} organisation'
  payload = {
  'org_id': organisation.id,
- 'site': str(get_current_site(request))
+ 'site': str(get_current_site(request)),
+ 'report_label': report_label,
  }
  recipient = request.user.email
- return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
+ return utils.make_async_email_report_task(
+ handle_email_report,
+ payload,
+ recipient,
+ REPORT_NAME,
+ hook='akvo.rsr.views.py_reports.utils.notify_user_on_failed_report'
+ )
 
 
 def handle_email_report(params, recipient):

akvo/rsr/views/py_reports/utils.py

@@ -11,14 +11,17 @@
 import os
 
 from collections import OrderedDict
-from datetime import date
+from datetime import date, datetime, timedelta
+from typing import cast
 from dateutil.parser import parse, ParserError
 from functools import cached_property
 from http import HTTPStatus
 
 from django.conf import settings
-from django.core.files.storage import FileSystemStorage, default_storage
+from django.core.files.storage import FileSystemStorage, Storage, default_storage
 from django.http import HttpResponse
+from django.utils import timezone
+from django_q.models import Task
 from django_q.tasks import async_task
 from weasyprint import HTML
 from weasyprint.fonts import FontConfiguration
@@ -28,21 +31,63 @@
 from akvo.rsr.project_overview import DisaggregationTarget, IndicatorType
 from akvo.rsr.models.result.utils import QUANTITATIVE, QUALITATIVE, PERCENTAGE_MEASURE, calculate_percentage
 from akvo.utils import ObjectReaderProxy, ensure_decimal, rsr_send_mail
+from akvo.utils.datetime import make_datetime_aware
 
+REPORTS_STORAGE_BASE_DIR = 'db/reports'
 
-def make_async_email_report_task(report_handler, payload, recipient, task_name):
- async_task(report_handler, payload, recipient, task_name=task_name)
+default_storage = cast(Storage, default_storage)
+
+
+def notify_user_on_failed_report(task: Task):
+ if task.success:
+ return
+ payload, recipient = task.args
+ user = User.objects.get(email=recipient)
+ report_label = payload.get('report_label', '')
+ if report_label:
+ report_label = f' ({report_label})'
+ rsr_send_mail(
+ [user.email],
+ subject='reports/email/failed_subject.txt',
+ message='reports/email/failed_message.txt',
+ msg_context={
+ 'username': user.get_full_name(),
+ 'report_label': report_label,
+ }
+ )
+ notify_dev_on_failed_task(task)
+
+
+def notify_dev_on_failed_task(task: Task):
+ if task.success:
+ return
+ recipient = getattr(settings, 'REPORT_ERROR_RECIPIENTS', [])
+ if not recipient:
+ return
+ rsr_send_mail(
+ recipient,
+ subject='reports/email/failed_subject_dev.txt',
+ message='reports/email/failed_message_dev.txt',
+ msg_context={'task': task}
+ )
+
+
+def make_async_email_report_task(report_handler, payload, recipient, task_name, hook=None):
+ hook = hook or notify_user_on_failed_report
+ async_task(report_handler, payload, recipient, task_name=task_name, hook=hook)
  return HttpResponse(
- 'Your report is being prepared. It will be sent to your email in a few moments.',
+ (
+ 'Your report is being generated. It will be sent to you over email. '
+ 'This can take several minutes depending on the amount of data needed to process.'
+ ),
  status=HTTPStatus.ACCEPTED,
  )
 
 
 def save_excel_and_send_email(workbook, site: str, user: User, filename='report.xlsx'):
  stream = io.BytesIO()
  workbook.save(stream)
- dir_path = "db/reports"
- file_url = save_report_file(dir_path, filename, stream.getvalue())
+ file_url = save_report_file(REPORTS_STORAGE_BASE_DIR, filename, stream.getvalue())
  rsr_send_mail(
  [user.email],
  subject='reports/email/subject.txt',
@@ -64,6 +109,19 @@ def save_report_file(dir_path: str, filename: str, content: bytes):
  return default_storage.url(file_path)
 
 
+def cleanup_expired_reports(now=None):
+ if not default_storage.exists(REPORTS_STORAGE_BASE_DIR):
+ return
+ now = now if isinstance(now, datetime) else timezone.now()
+ target_time = make_datetime_aware(now - timedelta(hours=24))
+ _, files = default_storage.listdir(REPORTS_STORAGE_BASE_DIR)
+ for file in files:
+ file_path = os.path.join(REPORTS_STORAGE_BASE_DIR, file)
+ created_at = default_storage.get_created_time(file_path)
+ if created_at < target_time:
+ default_storage.delete(file_path)
+
+
 def send_pdf_report(html, recipient, filename='reports.pdf'):
  font_config = FontConfiguration()
  pdf = HTML(string=html).write_pdf(font_config=font_config)

akvo/settings/90-finish.conf

@@ -129,4 +129,11 @@ AKVO_JOBS = {
  "task_name": "execute_aggregation_jobs",
  },
  },
+ "cleanup_expired_reports": {
+ "func": "akvo.rsr.views.py_reports.utils.cleanup_expired_reports",
+ "cron": "0 * * * *",
+ "kwargs": {
+ "task_name": "cleanup_expired_reports",
+ }
+ },
 }

akvo/templates/reports/email/failed_message.txt

@@ -0,0 +1,8 @@
+Dear {{ username }},
+
+We regret to inform you that your recent report generation request{{ report_label }} encountered an issue. Our developers have been notified and are actively working on a resolution.
+
+Please be patient, and feel free to retry generating the report after some time. We apologize for any inconvenience caused and appreciate your understanding.
+
+Thank you for your patience.
+Akvo.org

akvo/templates/reports/email/failed_message_dev.txt

@@ -0,0 +1,11 @@
+Failed running background job
+
+id: {{ task.id }}
+name: {{ task.name }}
+started: {{ task.started }}
+stopped: {{ task.stopped }}
+args: {{ task.args|safe }}
+kwargs: {{ task.kwargs|safe }}
+
+result:
+{{ task.result|safe }}

akvo/templates/reports/email/failed_subject.txt

@@ -0,0 +1 @@
+Report Generation Request Failed

akvo/templates/reports/email/failed_subject_dev.txt

@@ -0,0 +1 @@
+Background Job Failed!

akvo/utils/datetime.py

@@ -1,6 +1,13 @@
 from datetime import datetime
+from django.utils import timezone
 
 
 def datetime_remove_time(datetime_: datetime) -> datetime:
  """Removes the time components from a datetime effectively making it a date"""
  return datetime_.replace(hour=0, minute=0, second=0)
+
+
+def make_datetime_aware(dt: datetime):
+ return timezone.make_aware(dt, timezone.get_current_timezone())\
+ if timezone.is_naive(dt)\
+ else dt

scripts/docker/dev/50-docker-local-dev.conf

@@ -86,3 +86,4 @@ FIXTURE_DIRS = [
 ]
 
 RSR_DEMO_REQUEST_TO_EMAILS = ['[email protected]']
+REPORT_ERROR_RECIPIENTS = ['[email protected]']