Merge pull request #908 from alphagov/dev

v0.6.154 - Use reporting instead of testing complete date in Case metrics

.gitignore

@@ -17,6 +17,7 @@ data/localstack-data/*
 filehash.txt
 20210823_cleaned_case_record.csv
 **/data/s3_files/*
+**/data/postgres-data-full-stack/*
 cases.csv
 temp_manifest.yml
 backup.sql

Makefile

@@ -61,11 +61,26 @@ test:
 int_test:
 	docker compose --file stack_tests/integration_tests/docker-compose.yml up --abort-on-container-exit
 
+local_stack_up:
+	docker compose --file docker-compose-full-stack.yml  up --build
+
+local_stack_make_admin:
+	docker compose exec web sh -c "echo \"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('[email protected]', '[email protected]', 'secret')\" | python manage.py shell"
+
+local_stack_cleanup:
+	rm -r data/postgres-data-full-stack
+
 deploy_prototype:
+	python aws_prototype/main.py -b up -fd
+
+deploy_prototype_no_flush:
 	python aws_prototype/main.py -b up
 
 breakdown_prototype:
 	python aws_prototype/main.py -b down
 
 new_account_prototype:
 	python aws_prototype/main.py -b newaccount
+
+prototype_reload_database:
+	python aws_prototype/main.py -b reload_database

accessibility_monitoring_platform/apps/common/metrics.py

@@ -1,4 +1,4 @@
-""" Utility functions for calculating metrics and charts """
+"""Utility functions for calculating metrics and charts"""
 
 from collections import OrderedDict
 from dataclasses import dataclass
@@ -219,12 +219,12 @@ def get_case_progress_metrics() -> list[ThirtyDayMetric]:
         ThirtyDayMetric(
             label="Tests completed",
             last_30_day_count=Case.objects.filter(
-                testing_details_complete_date__gte=thirty_days_ago
+                reporting_details_complete_date__gte=thirty_days_ago
             ).count(),
             previous_30_day_count=Case.objects.filter(
-                testing_details_complete_date__gte=sixty_days_ago
+                reporting_details_complete_date__gte=sixty_days_ago
             )
-            .filter(testing_details_complete_date__lt=thirty_days_ago)
+            .filter(reporting_details_complete_date__lt=thirty_days_ago)
             .count(),
         ),
         ThirtyDayMetric(
@@ -258,7 +258,7 @@ def get_case_yearly_metrics() -> list[YearlyMetric]:
     start_date: datetime = get_first_of_this_month_last_year()
     for label, date_column_name in [
         ("Cases created", "created"),
-        ("Tests completed", "testing_details_complete_date"),
+        ("Tests completed", "reporting_details_complete_date"),
         ("Reports sent", "report_sent_date"),
         ("Cases completed", "completed_date"),
     ]:

accessibility_monitoring_platform/apps/common/tests/test_metrics.py

@@ -321,25 +321,25 @@ def test_get_case_progress_metrics(mock_date):
 
     Case.objects.create(
         created=datetime(2021, 11, 5, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         completed_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
     )
     Case.objects.create(
         created=datetime(2021, 12, 5, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         completed_date=datetime(2021, 12, 5, tzinfo=timezone.utc),
     )
     Case.objects.create(
         created=datetime(2021, 12, 6, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2021, 12, 6, tzinfo=timezone.utc),
         completed_date=datetime(2021, 12, 6, tzinfo=timezone.utc),
     )
     Case.objects.create(
         created=datetime(2022, 1, 1, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2021, 12, 6, tzinfo=timezone.utc),
         completed_date=datetime(2021, 12, 16, tzinfo=timezone.utc),
     )
@@ -371,25 +371,25 @@ def test_get_case_yearly_metrics(mock_datetime):
 
     Case.objects.create(
         created=datetime(2021, 11, 5, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         completed_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
     )
     Case.objects.create(
         created=datetime(2021, 12, 5, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         completed_date=datetime(2021, 12, 5, tzinfo=timezone.utc),
     )
     Case.objects.create(
         created=datetime(2021, 12, 6, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2021, 12, 6, tzinfo=timezone.utc),
         completed_date=datetime(2021, 12, 6, tzinfo=timezone.utc),
     )
     Case.objects.create(
         created=datetime(2022, 1, 1, tzinfo=timezone.utc),
-        testing_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
+        reporting_details_complete_date=datetime(2022, 1, 1, tzinfo=timezone.utc),
         report_sent_date=datetime(2021, 12, 6, tzinfo=timezone.utc),
         completed_date=datetime(2021, 12, 16, tzinfo=timezone.utc),
     )

accessibility_monitoring_platform/apps/common/tests/test_sitemap.py

@@ -857,7 +857,6 @@ def test_case_sitemap(rf):
 def test_page_name(url, expected_page_name, admin_client):
     """
     Test that the page renders and its name is as expected.
-    Also that an issue on that page can be reported.
     """
     case: Case = Case.objects.create()
     audit: Audit = Audit.objects.create(case=case)
@@ -878,16 +877,6 @@ def test_page_name(url, expected_page_name, admin_client):
 
     assertContains(response, expected_page_name)
 
-    issue_report_response: HttpResponse = admin_client.get(
-        f"/common/report-issue/?page_url={url}"
-    )
-
-    assert issue_report_response.status_code == 200
-
-    assertContains(response, "Report an issue")
-    assertContains(response, url)
-    assertContains(response, expected_page_name)
-
 
 def test_populate_subpage_instances():
     """Test populate_subpage_instances"""

accessibility_monitoring_platform/apps/common/tests/test_views.py

@@ -311,7 +311,7 @@ def test_issue_report_link(prototype_name, issue_report_link_expected, admin_cli
     "case_field, metric_id, lowercase_label",
     [
         ("created", "cases-created", "cases created"),
-        ("testing_details_complete_date", "tests-completed", "tests completed"),
+        ("reporting_details_complete_date", "tests-completed", "tests completed"),
         ("report_sent_date", "reports-sent", "reports sent"),
         ("completed_date", "cases-closed", "cases closed"),
     ],
@@ -353,7 +353,7 @@ def test_case_progress_metric_over(
     "case_field, metric_id, lowercase_label",
     [
         ("created", "cases-created", "cases created"),
-        ("testing_details_complete_date", "tests-completed", "tests completed"),
+        ("reporting_details_complete_date", "tests-completed", "tests completed"),
         ("report_sent_date", "reports-sent", "reports sent"),
         ("completed_date", "cases-closed", "cases closed"),
     ],
@@ -398,7 +398,7 @@ def test_case_progress_metric_under(
         (
             "Tests completed",
             "tests-completed-over-the-last-year",
-            "testing_details_complete_date",
+            "reporting_details_complete_date",
         ),
         ("Reports sent", "reports-sent-over-the-last-year", "report_sent_date"),
         ("Cases completed", "cases-completed-over-the-last-year", "completed_date"),
@@ -1271,3 +1271,30 @@ def test_navbar_tasks_emboldened(admin_client, admin_user):
         </li>""",
         html=True,
     )
+
+
+@pytest.mark.parametrize(
+    "url, expected_page_name",
+    [
+        ("/", "Your cases"),
+        ("/cases/1/edit-case-metadata/", "Case metadata"),
+        ("/audits/1/edit-audit-metadata/", "Initial test metadata"),
+    ],
+)
+def test_page_name(url, expected_page_name, admin_client):
+    """
+    Test that the page renders and problem page's url and name are populated
+    as expected.
+    """
+    case: Case = Case.objects.create()
+    Audit.objects.create(case=case)
+
+    response: HttpResponse = admin_client.get(
+        f"/common/report-issue/?page_url={url}&page_title={expected_page_name}"
+    )
+
+    assert response.status_code == 200
+
+    assertContains(response, "Report an issue")
+    assertContains(response, url)
+    assertContains(response, expected_page_name)

accessibility_monitoring_platform/apps/exports/csv_export_utils.py

@@ -9,13 +9,15 @@
 from typing import Any, Literal
 
 from django.db.models import QuerySet
-from django.http import HttpResponse
+from django.http import HttpResponse, StreamingHttpResponse
 from django.urls import reverse
 
 from ..audits.models import Audit
 from ..cases.models import Case, CaseCompliance, CaseStatus, Contact
 from ..reports.models import Report
 
+DOWNLOAD_CASES_CHUNK_SIZE: int = 500
+
 
 @dataclass
 class CSVColumn:
@@ -387,7 +389,7 @@ def required_data_missing(self):
     CSVColumn(
         column_header="Testing details page complete",
         source_class=Case,
-        source_attr="testing_details_complete_date",
+        source_attr="reporting_details_complete_date",
     ),
     CSVColumn(
         column_header="Link to report draft",
@@ -887,21 +889,35 @@ def populate_csv_columns(
 
 def download_cases(cases: QuerySet[Case], filename: str = "cases.csv") -> HttpResponse:
     """Given a Case queryset, download the data in csv format"""
-    response: HttpResponse = HttpResponse(content_type="text/csv")
-    response["Content-Disposition"] = f"attachment; filename={filename}"
 
-    writer: Any = csv.writer(response)
-    writer.writerow([column.column_header for column in CASE_COLUMNS_FOR_EXPORT])
+    class DummyFile:
+        def write(self, value_to_write):
+            return value_to_write
 
-    output: list[list[str]] = []
-    for case in cases:
-        case_columns: list[CSVColumn] = populate_csv_columns(
-            case=case, column_definitions=CASE_COLUMNS_FOR_EXPORT
-        )
-        row = [column.formatted_data for column in case_columns]
-        output.append(row)
-    writer.writerows(output)
+    def get_csv_output(cases: QuerySet[Case]) -> list[Any]:
+        writer: Any = csv.writer(DummyFile())
+        column_row: list[str] = [
+            column.column_header for column in CASE_COLUMNS_FOR_EXPORT
+        ]
 
+        output: str = writer.writerow(column_row)
+
+        for counter, case in enumerate(cases):
+            case_columns: list[CSVColumn] = populate_csv_columns(
+                case=case, column_definitions=CASE_COLUMNS_FOR_EXPORT
+            )
+            row = [column.formatted_data for column in case_columns]
+            output += writer.writerow(row)
+            if counter % DOWNLOAD_CASES_CHUNK_SIZE == 0:
+                yield output
+                output = ""
+        if output:
+            yield output
+
+    response = StreamingHttpResponse(
+        get_csv_output(cases=cases), content_type="text/csv"
+    )
+    response["Content-Disposition"] = f"attachment; filename={filename}"
     return response
 
 

accessibility_monitoring_platform/apps/exports/tests/test_csv_export_utils.py

@@ -8,7 +8,7 @@
 from typing import Any
 
 import pytest
-from django.http import HttpResponse
+from django.http import HttpResponse, StreamingHttpResponse
 
 from ...audits.models import Audit
 from ...cases.models import Case, Contact
@@ -54,9 +54,17 @@
 CONTACT_EMAIL: str = "[email protected]"
 
 
-def decode_csv_response(response: HttpResponse) -> tuple[list[str], list[list[str]]]:
+def decode_csv_response(
+    response: HttpResponse | StreamingHttpResponse,
+) -> tuple[list[str], list[list[str]]]:
     """Decode CSV HTTP response and break into column names and data"""
-    content: str = response.content.decode("utf-8")
+    if isinstance(response, StreamingHttpResponse):
+        content_chunks: list[str] = [
+            chunk.decode("utf-8") for chunk in response.streaming_content
+        ]
+        content: str = "".join(content_chunks)
+    else:
+        content: str = response.content.decode("utf-8")
     csv_reader: Any = csv.reader(io.StringIO(content))
     csv_body: list[list[str]] = list(csv_reader)
     csv_header: list[str] = csv_body.pop(0)

accessibility_monitoring_platform/settings/base.py

@@ -294,7 +294,7 @@
 CSP_IMG_SRC = ("'self'", "data:")
 
 AWS_PROTOTYPE_FILE: Path = Path("aws_prototype.json")
-if AWS_PROTOTYPE_FILE.exists():
+if AWS_PROTOTYPE_FILE.exists() and UNDER_TEST is False:
     aws_prototype_text: str = AWS_PROTOTYPE_FILE.read_text()
     aws_prototype_data: dict = json.loads(aws_prototype_text)
     AMP_PROTOTYPE_NAME = aws_prototype_data["prototype_name"]

amp_platform.DockerFile

@@ -16,7 +16,7 @@ RUN npm install
 EXPOSE 8001
 CMD make static_files_process \
     && python manage.py collectstatic --noinput \
-    && python manage.py clearsessions \
     && python manage.py migrate \
     && python manage.py recache_statuses \
+    && python manage.py clearsessions \
     && waitress-serve --port=8001 --threads=5 accessibility_monitoring_platform.wsgi:application