Merge pull request #5315 from akvo/org-reports-background-jobs

Use background jobs to generate Org level reports

akvo/rsr/tests/views/py_reports/test_send_report_via_email.py

@@ -12,13 +12,17 @@
  results_indicators_with_map_pdf_reports,
  nuffic_country_level_map_report,
  eutf_org_results_table_excel_report,
+ results_indicators_excel_report,
+ organisation_data_quality_overview_report,
+ organisation_projects_overview_report,
 )
 
 
 class SendReportViaEmailTestCase(BaseTestCase):
 
  def setUp(self):
  super().setUp()
+ self.org = self.create_organisation('Acme')
  self.program = self.create_program('Test program')
  self.user = self.create_user('[email protected]', 'password', is_admin=True)
  Country.objects.get_or_create(iso_code='nl')
@@ -57,7 +61,7 @@ def setUp(self):
  eutf_org_results_table_excel_report.handle_email_report,
  ),
  ])
- def test_send_report_via_djangoq_email(self, url_name, query_params, report_name, email_handler):
+ def test_send_program_reports_via_djangoq_email(self, url_name, query_params, report_name, email_handler):
  self.c.get(f"{reverse(url_name, args=(self.program.id,))}?{query_params}")
 
  # Check that the task was enqueued with django-q
@@ -81,3 +85,55 @@ def test_send_report_via_djangoq_email(self, url_name, query_params, report_name
  # Ensure an email was sent out
  msg = mail.outbox[0]
  self.assertEqual([self.user.email], msg.to)
+
+ @parameterized.expand([
+ (
+ 'py-reports-organisation-results-indicators-table', '',
+ results_indicators_excel_report.REPORT_NAME,
+ results_indicators_excel_report.handle_email_report,
+ ),
+ (
+ 'py-reports-organisation-data-quality-overview', 'format=pdf',
+ organisation_data_quality_overview_report.REPORT_NAME,
+ organisation_data_quality_overview_report.handle_email_report,
+ ),
+ (
+ 'py-reports-organisation-data-quality-overview', 'format=excel',
+ organisation_data_quality_overview_report.REPORT_NAME,
+ organisation_data_quality_overview_report.handle_email_report,
+ ),
+ (
+ 'py-reports-organisation-projects-overview', 'format=pdf',
+ organisation_projects_overview_report.REPORT_NAME,
+ organisation_projects_overview_report.handle_email_report,
+ ),
+ (
+ 'py-reports-organisation-projects-overview', 'format=excel',
+ organisation_projects_overview_report.REPORT_NAME,
+ organisation_projects_overview_report.handle_email_report,
+ ),
+ ])
+ def test_send_org_reports_via_djangoq_email(self, url_name, query_params, report_name, email_handler):
+ self.c.get(f"{reverse(url_name, args=(self.org.id,))}?{query_params}")
+
+ # Check that the task was enqueued with django-q
+ enqueued_task = OrmQ.objects.first()
+ self.assertIsNotNone(enqueued_task)
+ task_dict = SignedPackage.loads(enqueued_task.payload)
+ self.assertEquals(task_dict.get("name"), report_name)
+
+ # And with the correct program
+ task_args = task_dict.get("args")
+ params_arg = next(iter(task_args), {})
+ self.assertEquals(self.org.id, params_arg.get("org_id"),
+ msg="The expected organisation ID isn't present in the task's first argument")
+
+ # Emulate executing the task without going through django-q
+ # There's currently no easy way to do so
+ f = task_dict.get("func")
+ self.assertEqual(f, email_handler)
+ f(*task_dict.get("args"), **task_dict.get("kwargs"))
+
+ # Ensure an email was sent out
+ msg = mail.outbox[0]
+ self.assertEqual([self.user.email], msg.to)

akvo/rsr/views/py_reports/__init__.py

@@ -24,15 +24,15 @@
 from .kickstart_word_report import render_report as render_kickstart_report
 from .eutf_narrative_word_report import render_report as render_eutf_narrative_word_report
 from .organisation_data_quality_overview_report import \
- render_report as render_organisation_data_quality_overview
+ add_email_report_job as render_organisation_data_quality_overview
 from .eutf_org_results_table_excel_report import \
  add_email_report_job as render_eutf_org_results_table_excel_report
 from .eutf_project_results_table_excel_report import \
  render_report as render_eutf_project_results_table_excel_report
 from .results_indicators_excel_report import \
- render_report as render_results_indicators_excel_report
+ add_email_report_job as render_results_indicators_excel_report
 from .organisation_projects_overview_report import \
- render_report as render_org_projects_overview_report
+ add_email_report_job as render_org_projects_overview_report
 from .program_overview_excel_report import add_email_report_job as add_program_overview_excel_report_email_job
 from .program_overview_pdf_report import add_email_report_job as add_program_overview_pdf_report_email_job
 from .program_period_labels_overview_pdf_report import add_email_report_job as add_program_period_labels_overview

akvo/rsr/views/py_reports/organisation_data_quality_overview_report.py

@@ -9,12 +9,10 @@
 from django.utils import timezone
 
 from akvo.rsr.models import Organisation, Project
-from akvo.rsr.decorators import with_download_indicator
 from dateutil.relativedelta import relativedelta
 from django.contrib.auth.decorators import login_required
 from django.conf import settings
 from django.db.models import Q
-from django.http import HttpResponse, HttpResponseBadRequest
 from django.shortcuts import get_object_or_404
 from django.template.loader import render_to_string
 from pyexcelerate import Workbook, Style, Font, Color, Alignment, Format
@@ -23,38 +21,46 @@
 
 from . import utils
 
+REPORT_NAME = 'organisation_data_quality'
+
 
 @login_required
-@with_download_indicator
-def render_report(request, org_id):
+def add_email_report_job(request, org_id):
  organisation = get_object_or_404(Organisation, pk=org_id)
+ payload = {
+ 'org_id': organisation.id,
+ 'format': request.GET.get('format')
+ }
+ recipient = request.user.email
+ return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
+
+
+def handle_email_report(params, recipient):
+ organisation = Organisation.objects.get(pk=params['org_id'])
+ format = params['format']
  now = timezone.now()
  reader = OrganisationDataQualityReader(organisation, now)
 
- format = request.GET.get('format')
-
  if format == 'pdf':
- return _render_pdf(reader, True if request.GET.get('show-html', '') else False)
+ html = _render_pdf(reader)
+ filename = '{}-{}-organisation-data-quality.pdf'.format(
+ reader.date.strftime('%Y%m%d'), reader.organisation.id)
+ utils.send_pdf_report(html, recipient, filename)
  elif format == 'excel':
- return _render_excel(reader)
+ wb = _render_excel(reader)
+ filename = '{}-{}-organisation-data-quality.xlsx'.format(
+ reader.date.strftime('%Y%m%d'), reader.organisation.id)
+ utils.send_excel_report(wb, recipient, filename)
  else:
- return HttpResponseBadRequest('Unsupported format.')
+ pass
 
 
-def _render_pdf(reader, show_html=True):
- html = render_to_string(
+def _render_pdf(reader):
+ return render_to_string(
  'reports/organisation-data-quality-overview.html',
  context={'reader': reader, 'domain': settings.RSR_DOMAIN}
  )
 
- if show_html:
- return HttpResponse(html)
-
- filename = '{}-{}-organisation-data-quality.pdf'.format(
- reader.date.strftime('%Y%m%d'), reader.organisation.id)
-
- return utils.make_pdf_response(html, filename)
-
 
 def _render_excel(reader):
  wb = Workbook()
@@ -262,10 +268,7 @@ def _render_excel(reader):
  ws.set_cell_style(row, i, table_footer_style)
  ws.set_cell_value(row, 1, len(reader.without_photo_list))
 
- filename = '{}-{}-organisation-data-quality.xlsx'.format(
- reader.date.strftime('%Y%m%d'), reader.organisation.id)
-
- return utils.make_excel_response(wb, filename)
+ return wb
 
 
 class OrganisationDataQualityReader(object):

akvo/rsr/views/py_reports/organisation_projects_overview_report.py

@@ -11,12 +11,10 @@
 from akvo.codelists.models import ActivityStatus
 from akvo.rsr.models import Organisation
 from akvo.utils import ObjectReaderProxy
-from akvo.rsr.decorators import with_download_indicator
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.db.models import Count, Sum
 from django.db.models.functions import Trunc
-from django.http import HttpResponse, HttpResponseBadRequest
 from django.shortcuts import get_object_or_404
 from django.template.loader import render_to_string
 from django.urls import reverse
@@ -151,40 +149,52 @@ def _activity_status_name(value, version=settings.IATI_VERSION):
  return status.name if status else 'None'
 
 
+REPORT_NAME = 'organisation_projects_overview'
+
+
 @login_required
-@with_download_indicator
-def render_report(request, org_id):
+def add_email_report_job(request, org_id):
  organisation = get_object_or_404(Organisation, pk=org_id)
- reader = OrganisationProjectsOverviewReader(organisation)
+ payload = {
+ 'org_id': organisation.id,
+ 'format': request.GET.get('format')
+ }
+ recipient = request.user.email
+ return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
+
 
- format = request.GET.get('format')
+def handle_email_report(params, recipient):
+ organisation = Organisation.objects.get(pk=params['org_id'])
+ format = params['format']
+ reader = OrganisationProjectsOverviewReader(organisation)
+ current_date = timezone.now()
 
  if format == 'pdf':
- return _render_pdf(reader, True if request.GET.get('show-html', '') else False)
+ html = _render_pdf(reader, current_date)
+ filename = '{}-{}-organisation-projects-overview.pdf'.format(
+ current_date.strftime('%Y%m%d'), reader.id)
+ utils.send_pdf_report(html, recipient, filename)
+
  elif format == 'excel':
- return _render_excel(reader)
+ wb = _render_excel(reader)
+
+ filename = '{}-{}-organisation-projects-overview.xlsx'.format(
+ current_date.strftime('%Y%m%d'), reader.id)
+ utils.send_excel_report(wb, recipient, filename)
+
  else:
- return HttpResponseBadRequest('Unsupported format.')
+ pass
 
 
-def _render_pdf(reader, show_html=True):
- current_date = timezone.now()
- html = render_to_string(
+def _render_pdf(reader, current_date):
+ return render_to_string(
  'reports/organisation-projects-overview.html',
  context={
  'reader': reader,
  'current_date': current_date
  }
  )
 
- if show_html:
- return HttpResponse(html)
-
- filename = '{}-{}-organisation-projects-overview.pdf'.format(
- current_date.strftime('%Y%m%d'), reader.id)
-
- return utils.make_pdf_response(html, filename)
-
 
 def _render_excel(reader):
  section_title_style = Style(font=Font(size=14, bold=True))
@@ -414,8 +424,4 @@ def _render_excel(reader):
  ws.set_cell_value(row, i, value)
 
  row += 1
-
- filename = '{}-{}-organisation-projects-overview.xlsx'.format(
- timezone.now().strftime('%Y%m%d'), reader.id)
-
- return utils.make_excel_response(wb, filename)
+ return wb

akvo/rsr/views/py_reports/results_indicators_excel_report.py

@@ -9,7 +9,6 @@
 
 from akvo.rsr.models import Organisation, IndicatorPeriod
 from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE
-from akvo.rsr.decorators import with_download_indicator
 from django.contrib.auth.decorators import login_required
 from django.db.models import Count
 from django.shortcuts import get_object_or_404
@@ -41,10 +40,28 @@ def build_view_object(organisation):
  return utils.make_project_proxies(periods)
 
 
+REPORT_NAME = 'organisation_results_indicators_table'
+
+
 @login_required
-@with_download_indicator
-def render_report(request, org_id):
+def add_email_report_job(request, org_id):
  organisation = get_object_or_404(Organisation, pk=org_id)
+ payload = {
+ 'org_id': organisation.id,
+ }
+ recipient = request.user.email
+ return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
+
+
+def handle_email_report(params, recipient):
+ organisation = Organisation.objects.get(pk=params['org_id'])
+ wb = generate_workbook(organisation)
+ filename = '{}-{}-results-and-indicators-simple-table.xlsx'.format(
+ timezone.now().strftime('%Y%m%d'), organisation.id)
+ utils.send_excel_report(wb, recipient, filename)
+
+
+def generate_workbook(organisation):
  projects = build_view_object(organisation)
 
  use_indicator_target = False
@@ -196,8 +213,4 @@ def render_report(request, org_id):
  ws.set_cell_value(row, 29, indicator.id)
  ws.set_cell_value(row, 30, period.id)
  row += 1
-
- filename = '{}-{}-results-and-indicators-simple-table.xlsx'.format(
- timezone.now().strftime('%Y%m%d'), organisation.id)
-
- return utils.make_excel_response(wb, filename)
+ return wb