Fix memory issue on org results indicators excel report

akvo/rsr/dataclasses.py

@@ -575,6 +575,7 @@ class IndicatorData(object):
  type: int = QUANTITATIVE
  measure: str = ''
  cumulative: bool = False
+ ascending: bool = False
  description: str = ''
  baseline_year: Optional[int] = None
  baseline_value: Optional[Decimal] = None
@@ -591,6 +592,7 @@ def make(cls, data, prefix=''):
  type=data.get(f"{prefix}type", QUANTITATIVE),
  measure=data.get(f"{prefix}measure", ''),
  cumulative=data.get(f"{prefix}cumulative", False),
+ ascending=data.get(f"{prefix}ascending", False),
  description=data.get(f"{prefix}description", ''),
  baseline_year=data.get(f"{prefix}baseline_year", None),
  baseline_value=maybe_decimal(data.get(f"{prefix}baseline_value", None)),
@@ -667,6 +669,44 @@ def get_codelist_name(code, version=settings.IATI_VERSION):
  return ''
 
 
+@dataclass(frozen=True)
+class ProjectData(object):
+ id: int
+ title: str
+ subtitle: str
+ iati_activity_id: str
+ date_start_planned: Optional[date]
+ date_end_planned: Optional[date]
+ date_start_actual: Optional[date]
+ date_end_actual: Optional[date]
+ targets_at: str
+ recipient_countries: List[str] = field(default_factory=list)
+ partners: List[str] = field(default_factory=list)
+ results: List[ResultData] = field(default_factory=list)
+
+ @classmethod
+ def make(cls, data, prefix=''):
+ return cls(
+ id=data[f"{prefix}id"],
+ title=data.get(f"{prefix}title", ''),
+ subtitle=data.get(f"{prefix}subtitle", ''),
+ iati_activity_id=data.get(f"{prefix}iati_activity_id", ''),
+ date_start_planned=data.get(f"{prefix}date_start_planned", None),
+ date_end_planned=data.get(f"{prefix}date_end_planned", None),
+ date_start_actual=data.get(f"{prefix}date_start_actual", None),
+ date_end_actual=data.get(f"{prefix}date_end_actual", None),
+ targets_at=data.get(f"{prefix}targets_at", 'period'),
+ )
+
+ @property
+ def country_codes(self):
+ return ', '.join({it.lower() for it in self.recipient_countries})
+
+ @property
+ def partner_names(self):
+ return ', '.join({it for it in self.partners})
+
+
 class IndicatorType(Enum):
  Quantitative = QUANTITATIVE
  Qualitative = QUALITATIVE

akvo/rsr/tests/views/py_reports/test_cumulative_updates.py

@@ -7,7 +7,6 @@
 from akvo.rsr.usecases.period_update_aggregation import aggregate
 from akvo.rsr.views.py_reports import (
  results_indicators_with_map_pdf_reports,
- results_indicators_excel_report,
  kickstart_word_report,
  eutf_org_results_table_excel_report,
  program_period_labels_overview_pdf_report,
@@ -149,11 +148,6 @@ def make_project_view(self, project: Project) -> utils.ProjectProxy:
  return results_indicators_with_map_pdf_reports.build_view_object(project, self.PERIOD_2_START, self.PERIOD_3_END)
 
 
-class ResultsIndicatorsExcelReportTestCase(ObjectReaderCumulativeUpdateBaseTestCase, BaseTestCase):
- def make_project_view(self, project: Project) -> utils.ProjectProxy:
- return results_indicators_excel_report.build_view_object(project.reporting_org)[0]
-
-
 class KickstartWordReportTestCase(ObjectReaderCumulativeUpdateBaseTestCase, BaseTestCase):
  def make_project_view(self, project: Project) -> utils.ProjectProxy:
  return kickstart_word_report.build_view_object(project, self.PERIOD_2_START, self.PERIOD_3_END)

akvo/rsr/views/py_reports/results_indicators_excel_report.py

@@ -5,25 +5,49 @@
 Akvo RSR module. For additional details on the GNU license please
 see < http://www.gnu.org/licenses/agpl.html >.
 """
+from django.conf import settings
 from django.utils import timezone
+from akvo.rsr.dataclasses import IndicatorData, PeriodData, ProjectData, ResultData
 
-from akvo.rsr.models import Organisation, IndicatorPeriod
+from akvo.rsr.models import Organisation, IndicatorPeriod, RecipientCountry, Partnership, Project
 from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE
 from django.contrib.auth.decorators import login_required
 from django.db.models import Count
 from django.shortcuts import get_object_or_404
 from pyexcelerate import Workbook, Style, Font, Color, Fill, Alignment
 
+from akvo.utils import ensure_decimal
+
 from . import utils
 
+REPORT_NAME = 'organisation_results_indicators_table'
+
+
+@login_required
+def add_email_report_job(request, org_id):
+ organisation = get_object_or_404(Organisation, pk=org_id)
+ payload = {
+ 'org_id': organisation.id,
+ }
+ recipient = request.user.email
+ return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
+
 
-def build_view_object(organisation):
+def handle_email_report(params, recipient):
+ organisation = Organisation.objects.get(pk=params['org_id'])
+ wb = generate_workbook(organisation)
+ filename = '{}-{}-results-and-indicators-simple-table.xlsx'.format(
+ timezone.now().strftime('%Y%m%d'), organisation.id)
+ utils.send_excel_report(wb, recipient, filename)
+
+
+def fetch_periods(organisation: Organisation):
  project_ids = organisation.all_projects()\
  .annotate(results_count=Count('results'))\
  .filter(results_count__gt=0)\
  .values_list('pk', flat=True)
 
- periods = IndicatorPeriod.objects\
+ queryset = IndicatorPeriod.objects\
  .select_related('indicator', 'indicator__result', 'indicator__result__project')\
  .filter(indicator__result__project__in=project_ids)\
  .exclude(indicator__result__type__iexact='')\
@@ -36,40 +60,128 @@ def build_view_object(organisation):
  'period_start',
  'period_end'
  )
+ return queryset.values(
+ 'id', 'period_start', 'period_end', 'target_value', 'target_comment', 'actual_value', 'actual_comment', 'narrative',
+ 'indicator__id', 'indicator__title', 'indicator__type', 'indicator__measure', 'indicator__cumulative',
+ 'indicator__ascending', 'indicator__description', 'indicator__baseline_year', 'indicator__baseline_value',
+ 'indicator__baseline_comment', 'indicator__target_value', 'indicator__target_comment',
+ 'indicator__result__id', 'indicator__result__title', 'indicator__result__description',
+ 'indicator__result__type', 'indicator__result__aggregation_status',
+ 'indicator__result__project__id', 'indicator__result__project__title', 'indicator__result__project__subtitle',
+ 'indicator__result__project__iati_activity_id', 'indicator__result__project__date_start_planned',
+ 'indicator__result__project__date_end_planned', 'indicator__result__project__date_start_actual',
+ 'indicator__result__project__date_end_actual', 'indicator__result__project__targets_at',
+ )
 
- return utils.make_project_proxies(periods)
 
+def get_view_objects(organisation):
+ raw_periods = fetch_periods(organisation)
+ lookup = {
+ 'projects': {},
+ 'results': {},
+ 'indicators': {},
+ 'periods': {},
+ }
+ for r in raw_periods:
+ project_id = r['indicator__result__project__id']
+ result_id = r['indicator__result__id']
+ indicator_id = r['indicator__id']
+ period_id = r['id']
+ if project_id not in lookup['projects']:
+ lookup['projects'][project_id] = ProjectData.make(r, 'indicator__result__project__')
+ project = lookup['projects'][project_id]
+ if result_id not in lookup['results']:
+ result = ResultData.make(r, 'indicator__result__')
+ project.results.append(result)
+ lookup['results'][result_id] = result
+ else:
+ result = lookup['results'][result_id]
+ result = lookup['results'][result_id]
+ if indicator_id not in lookup['indicators']:
+ indicator = IndicatorData.make(r, 'indicator__')
+ result.indicators.append(indicator)
+ lookup['indicators'][indicator_id] = indicator
+ else:
+ indicator = lookup['indicators'][indicator_id]
+ if period_id not in lookup['periods']:
+ period = PeriodData.make(r)
+ indicator.periods.append(period)
+ lookup['periods'][period_id] = period
+ project_ids = {id for id in lookup['projects'].keys()}
+ recipient_countries = RecipientCountry.objects.filter(project__in=project_ids).values('project', 'country')
+ for recipient in recipient_countries:
+ project = lookup['projects'][recipient['project']]
+ project.recipient_countries.append(recipient['country'])
+ partners = Partnership.objects.filter(project__in=project_ids)\
+ .exclude(organisation__isnull=True)\
+ .values('project', 'organisation__name')
+ for partner in partners:
+ project = lookup['projects'][partner['project']]
+ project.partners.append(partner['organisation__name'])
+ return [it for it in lookup['projects'].values()]
 
-REPORT_NAME = 'organisation_results_indicators_table'
 
+def is_using_indicator_target(projects):
+ for project in projects:
+ program = Project.objects.get(id=project.id).get_program()
+ targets_at = program.targets_at if program else project.targets_at
+ if targets_at == 'indicator':
+ return True
+ return False
 
-@login_required
-def add_email_report_job(request, org_id):
- organisation = get_object_or_404(Organisation, pk=org_id)
- payload = {
- 'org_id': organisation.id,
- }
- recipient = request.user.email
- return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
 
+def get_eutf_members():
+ try:
+ root = Project.objects.get(id=settings.EUTF_ROOT_PROJECT)
+ return root.descendants().values_list('id', flat=True)
+ except Project.DoesNotExist:
+ return []
 
-def handle_email_report(params, recipient):
- organisation = Organisation.objects.get(pk=params['org_id'])
- wb = generate_workbook(organisation)
- filename = '{}-{}-results-and-indicators-simple-table.xlsx'.format(
- timezone.now().strftime('%Y%m%d'), organisation.id)
- utils.send_excel_report(wb, recipient, filename)
+
+def get_period_start(period, project, in_eutf_hierarchy):
+ if not in_eutf_hierarchy:
+ return period.period_start
+
+ if project.id == settings.EUTF_ROOT_PROJECT:
+ return period.period_start
+
+ if project.date_start_actual:
+ return project.date_start_actual
+
+ return project.date_start_planned
+
+
+def get_period_end(period, project, in_eutf_hierarchy):
+ if not in_eutf_hierarchy:
+ return period.period_end
+
+ if project.id == settings.EUTF_ROOT_PROJECT:
+ return period.period_end
+
+ if project.date_end_actual:
+ return project.date_end_actual
+
+ return project.date_end_planned
+
+
+def get_total_period_targets(indicator):
+ total = 0
+ for period in indicator.periods:
+ total += ensure_decimal(period.target_value)
+ return total
+
+
+def get_indicator_target(indicator, targets_at):
+ return ensure_decimal(indicator.target_value) if targets_at == 'indicator' else get_total_period_targets(indicator)
 
 
 def generate_workbook(organisation):
- projects = build_view_object(organisation)
+ projects = get_view_objects(organisation)
+ use_indicator_target = is_using_indicator_target(projects)
+ eutf_members = get_eutf_members()
 
- use_indicator_target = False
- for project in projects:
- if project.use_indicator_target:
- print(project.id, project.use_indicator_target)
- use_indicator_target = True
- break
+ def in_eutf_hierarchy(project):
+ return project.id in eutf_members
 
  wb = Workbook()
  ws = wb.new_sheet('ProjectList')
@@ -191,19 +303,19 @@ def generate_workbook(organisation):
  col = 17
  if use_indicator_target:
  col += 1
- ws.set_cell_value(row, col, indicator.target_value or ' ')
+ ws.set_cell_value(row, col, get_indicator_target(indicator, project.targets_at) or ' ')
  col += 1
  ws.set_cell_value(row, col, indicator.target_comment or ' ')
  col += 1
- ws.set_cell_value(row, col, utils.get_period_start(period, project.in_eutf_hierarchy) or ' ')
+ ws.set_cell_value(row, col, get_period_start(period, project, in_eutf_hierarchy(project)) or ' ')
  col += 1
- ws.set_cell_value(row, col, utils.get_period_end(period, project.in_eutf_hierarchy) or ' ')
+ ws.set_cell_value(row, col, get_period_end(period, project, in_eutf_hierarchy(project)) or ' ')
  if not use_indicator_target:
  col += 1
  ws.set_cell_value(row, col, period.target_value or ' ')
  col += 1
  ws.set_cell_value(row, col, period.target_comment or ' ')
- ws.set_cell_value(row, 22, period.actual_value or ' ')
+ ws.set_cell_value(row, 22, period.period_actual_value or ' ')
  ws.set_cell_value(row, 23, period.actual_comment or ' ')
  ws.set_cell_value(row, 24, project.country_codes or ' ')
  ws.set_cell_value(row, 25, result.iati_type_name or ' ')