diff --git a/project/.pylintrc b/project/.pylintrc index d4a615ef..3a59c369 100644 --- a/project/.pylintrc +++ b/project/.pylintrc @@ -128,7 +128,10 @@ disable=parameter-unpacking, dict-values-not-iterating, fixme, bad-continuation, - no-member + no-member, + invalid-name, + too-many-locals, + no-name-in-module, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/project/api/admin.py b/project/api/admin.py index f82575c6..6369ddf2 100644 --- a/project/api/admin.py +++ b/project/api/admin.py @@ -18,6 +18,7 @@ """ from django.contrib import admin +from simple_history.admin import SimpleHistoryAdmin from .models import ( TerritorialEntity, @@ -35,15 +36,15 @@ ) # Register your models here. -admin.site.register(TerritorialEntity) -admin.site.register(PoliticalRelation) -admin.site.register(SpacetimeVolume) -admin.site.register(CachedData) -admin.site.register(Narration) -admin.site.register(Narrative) -admin.site.register(MapSettings) -admin.site.register(City) -admin.site.register(Profile) -admin.site.register(NarrativeVote) -admin.site.register(Symbol) -admin.site.register(SymbolFeature) +admin.site.register(TerritorialEntity, SimpleHistoryAdmin) +admin.site.register(PoliticalRelation, SimpleHistoryAdmin) +admin.site.register(SpacetimeVolume, SimpleHistoryAdmin) +admin.site.register(CachedData, SimpleHistoryAdmin) +admin.site.register(Narration, SimpleHistoryAdmin) +admin.site.register(Narrative, SimpleHistoryAdmin) +admin.site.register(MapSettings, SimpleHistoryAdmin) +admin.site.register(City, SimpleHistoryAdmin) +admin.site.register(Profile, SimpleHistoryAdmin) +admin.site.register(NarrativeVote, SimpleHistoryAdmin) +admin.site.register(Symbol, SimpleHistoryAdmin) +admin.site.register(SymbolFeature, SimpleHistoryAdmin) diff --git a/project/api/apps.py b/project/api/apps.py index cc685b22..cd810664 100644 --- a/project/api/apps.py +++ b/project/api/apps.py @@ -18,6 +18,17 @@ """ from django.apps import AppConfig +from simple_history.signals import pre_create_historical_record + + +def add_group_to_historical_record(sender, **kwargs): # pylint: disable=W0613 + """ + Save group to historical record + """ + instance = kwargs["instance"] + if getattr(instance, "group", None): + kwargs["history_instance"].group = instance.group + del instance.group class ApiConfig(AppConfig): @@ -26,3 +37,10 @@ class ApiConfig(AppConfig): """ name = "api" + + def ready(self): + from .models import HistoricalSpacetimeVolume # pylint: disable=C0415 + + pre_create_historical_record.connect( + add_group_to_historical_record, sender=HistoricalSpacetimeVolume + ) diff --git a/project/api/migrations/0022_historicalspacetimevolume_group.py b/project/api/migrations/0022_historicalspacetimevolume_group.py new file mode 100644 index 00000000..cc48cacb --- /dev/null +++ b/project/api/migrations/0022_historicalspacetimevolume_group.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.9 on 2020-03-15 06:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0021_merge_20200213_1800'), + ] + + operations = [ + migrations.AddField( + model_name='historicalspacetimevolume', + name='group', + field=models.PositiveIntegerField(default=0), + preserve_default=False, + ), + ] diff --git a/project/api/migrations/0023_auto_20200315_0628.py b/project/api/migrations/0023_auto_20200315_0628.py new file mode 100644 index 00000000..1a6e9d35 --- /dev/null +++ b/project/api/migrations/0023_auto_20200315_0628.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.9 on 2020-03-15 06:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0022_historicalspacetimevolume_group'), + ] + + operations = [ + migrations.AlterField( + model_name='historicalspacetimevolume', + name='group', + field=models.PositiveIntegerField(blank=True, null=True), + ), + ] diff --git a/project/api/migrations/0024_merge_20200330_1854.py b/project/api/migrations/0024_merge_20200330_1854.py new file mode 100644 index 00000000..277e37af --- /dev/null +++ b/project/api/migrations/0024_merge_20200330_1854.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.11 on 2020-03-30 18:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0023_auto_20200315_0628'), + ('api', '0023_mapcolorscheme'), + ] + + operations = [ + ] diff --git a/project/api/models.py b/project/api/models.py index 16c7826f..0d27aead 100644 --- a/project/api/models.py +++ b/project/api/models.py @@ -292,6 +292,18 @@ def save(self, *args, **kwargs): # pylint: disable=W0221 super(City, self).save(*args, **kwargs) +class OverlapGroupedHistoricalModel(models.Model): + """ + Abstract model for history models in which + the territory changed due to an overlap + """ + + group = models.PositiveIntegerField(null=True, blank=True) + + class Meta: + abstract = True + + class SpacetimeVolume(models.Model): """ Maps a set of Territories to a TerritorialEntity at a specific time @@ -306,7 +318,7 @@ class SpacetimeVolume(models.Model): references = ArrayField(models.TextField(max_length=500), blank=True, null=True) visual_center = models.PointField(blank=True, null=True) related_events = models.ManyToManyField(CachedData, blank=True) - history = HistoricalRecords() + history = HistoricalRecords(bases=[OverlapGroupedHistoricalModel,]) def calculate_center(self): """ @@ -364,11 +376,22 @@ def clean(self, *args, **kwargs): # pylint: disable=W0221 super(SpacetimeVolume, self).clean(*args, **kwargs) def save(self, *args, **kwargs): # pylint: disable=W0221 - self.full_clean() + self.full_clean(exclude=["id"]) super(SpacetimeVolume, self).save(*args, **kwargs) if not self.visual_center: self.calculate_center() + def save_without_historical_record(self, *args, **kwargs): + """ + Saves the model without creating a new historical record + """ + self.skip_history_when_saving = True # pylint: disable=W0201 + try: + ret = self.save(*args, **kwargs) # pylint: disable=E1111 + finally: + del self.skip_history_when_saving + return ret + class Narrative(models.Model): """ diff --git a/project/api/serializers.py b/project/api/serializers.py index 286ed23e..e7eb67e0 100644 --- a/project/api/serializers.py +++ b/project/api/serializers.py @@ -41,6 +41,8 @@ Narration, NarrativeVote, Profile, + HistoricalSpacetimeVolume, + HistoricalTerritorialEntity, ) @@ -263,3 +265,95 @@ def get_narration_count(self, obj): # pylint: disable=R0201 """ return obj.narration_set.count() + + +class StvHistoryListSerializer(ModelSerializer): + """ + Serializes the HistoricalSpacetimeVolume model + """ + + history_user = SerializerMethodField() + history_user_id = SerializerMethodField() + + def get_history_user(self, obj): # pylint: disable=R0201 + """ + Returns the username of the user who made the change. + """ + + if obj.history_user is not None: + return obj.history_user.username + return None + + def get_history_user_id(self, obj): # pylint: disable=R0201 + """ + Returns the id of the user who made the change. + """ + + if obj.history_user is not None: + return obj.history_user.id + return None + + class Meta: + model = HistoricalSpacetimeVolume + exclude = ["territory"] + + +class StvHistoryRetrieveSerializer(ModelSerializer): + """ + Serializes the HistoricalSpacetimeVolume model + """ + + history_user = SerializerMethodField() + history_user_id = SerializerMethodField() + + def get_history_user(self, obj): # pylint: disable=R0201 + """ + Returns the username of the user who made the change. + """ + + if obj.history_user is not None: + return obj.history_user.username + return None + + def get_history_user_id(self, obj): # pylint: disable=R0201 + """ + Returns the id of the user who made the change. + """ + + if obj.history_user is not None: + return obj.history_user.id + return None + + class Meta: + model = HistoricalSpacetimeVolume + fields = "__all__" + + +class TeHistorySerializer(ModelSerializer): + """ + Serializes the HistoricalTerritorialEntity model + """ + + history_user = SerializerMethodField() + history_user_id = SerializerMethodField() + + def get_history_user(self, obj): # pylint: disable=R0201 + """ + Returns the username of the user who made the change. + """ + + if obj.history_user is not None: + return obj.history_user.username + return None + + def get_history_user_id(self, obj): # pylint: disable=R0201 + """ + Returns the id of the user who made the change. + """ + if obj.history_user is not None: + return obj.history_user.id + return None + + class Meta: + model = HistoricalTerritorialEntity + fields = "__all__" diff --git a/project/api/tests/stv_tests.py b/project/api/tests/stv_tests.py index a2bdfd0b..907fb135 100644 --- a/project/api/tests/stv_tests.py +++ b/project/api/tests/stv_tests.py @@ -166,3 +166,26 @@ def test_api_can_not_create_stv(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) response = self.client.post(url, data_overlapping) self.assertEqual(response.status_code, status.HTTP_409_CONFLICT) + + @authorized + def test_api_can_query_stv_history(self): + """ + Ensure we can query for all Cities + """ + + url = reverse("stv-history-list") + response = self.client.get(url, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data[0]["id"], 11) + + @authorized + def test_api_can_query_te_history_detail(self): + """ + Ensure we can query for all Cities + """ + + url = reverse("stv-history-detail", args=[11]) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["id"], 11) diff --git a/project/api/tests/te_tests.py b/project/api/tests/te_tests.py index 88c227ec..7d7a6c84 100644 --- a/project/api/tests/te_tests.py +++ b/project/api/tests/te_tests.py @@ -91,3 +91,31 @@ def test_api_can_query_te(self): response = self.client.get(url, format="json") self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["admin_level"], 1) + + @authorized + def test_api_can_query_te_history(self): + """ + Ensure we can query for all Cities + """ + + url = reverse("te-history-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data[0]["label"], "France") + + params = {"entity": "105"} + response = self.client.get(url, params, format="json") + + for record in response.data: + self.assertEqual(record["id"], 105) + + @authorized + def test_api_can_query_te_history_detail(self): + """ + Ensure we can query for all Cities + """ + + url = reverse("te-history-detail", args=[126]) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["label"], "France") diff --git a/project/api/urls.py b/project/api/urls.py index d0996c7b..0cefe0c4 100644 --- a/project/api/urls.py +++ b/project/api/urls.py @@ -36,6 +36,9 @@ ROUTER.register(r"profiles", views.ProfileViewSet) ROUTER.register(r"symbols", views.SymbolViewSet) ROUTER.register(r"symbol-features", views.SymbolFeatureViewSet) +ROUTER.register(r"stv-history", views.StvHistoryViewSet, basename="stv-history") +ROUTER.register(r"te-history", views.TeHistoryViewSet, basename="te-history") + urlpatterns = [ path( @@ -55,6 +58,7 @@ ), path("mvt/stv///", views.mvt_stv, name="mvt-stv"), path("spacetime-volumes//download", views.stv_downloader), + path("stv-history//download", views.historical_stv_downloader), path("territorial-entities/list", views.te_list), path("", include(ROUTER.urls)), ] diff --git a/project/api/views/endpoints/stv_downloader.py b/project/api/views/endpoints/stv_downloader.py index a9e755ea..5e620ef9 100644 --- a/project/api/views/endpoints/stv_downloader.py +++ b/project/api/views/endpoints/stv_downloader.py @@ -22,18 +22,14 @@ from django.core.serializers import serialize from django.http import HttpResponse, JsonResponse from jdcal import jd2gcal -from api.models import SpacetimeVolume +from api.models import SpacetimeVolume, HistoricalSpacetimeVolume -def stv_downloader(request, primary_key): +def stv_to_geojson_response(stv): """ - Download stvs as geojson. + Function for serializing stv queryset with one member to geojson. """ - stv = SpacetimeVolume.objects.filter(pk=primary_key) - if len(stv) == 0: - return HttpResponse(status=404) - geojson = serialize( "geojson", stv, @@ -51,11 +47,12 @@ def stv_downloader(request, primary_key): geojson = json.loads(geojson) for features in geojson["features"]: - features["properties"]["visual_center"] = { - "type": "Feature", - "properties": None, - "geometry": json.loads(stv[0].visual_center.json), - } + if not stv[0].visual_center is None: + features["properties"]["visual_center"] = { + "type": "Feature", + "properties": None, + "geometry": json.loads(stv[0].visual_center.json), + } features["properties"]["entity"] = { "label": stv[0].entity.label, "pk": stv[0].entity.pk, @@ -66,10 +63,34 @@ def stv_downloader(request, primary_key): end_date = jd2gcal(stv[0].end_date, 0) end_string = "{}-{}-{}".format(end_date[0], end_date[1], end_date[2]) + response = JsonResponse(geojson) response["Content-Disposition"] = "attachment;filename={}_{}_{}.json;".format( stv[0].entity.label, start_string, end_string ) - return response + + +def stv_downloader(request, primary_key): + """ + Download stvs as geojson. + """ + + stv = SpacetimeVolume.objects.filter(pk=primary_key) + if len(stv) == 0: + return HttpResponse(status=404) + + return stv_to_geojson_response(stv) + + +def historical_stv_downloader(request, primary_key): + """ + Download historical stvs as geojson. + """ + + history = HistoricalSpacetimeVolume.objects.filter(history_id=primary_key) + if len(history) == 0: + return HttpResponse(status=404) + + return stv_to_geojson_response(history) diff --git a/project/api/views/stv_view.py b/project/api/views/stv_view.py index 8ab455bc..a591c8e4 100644 --- a/project/api/views/stv_view.py +++ b/project/api/views/stv_view.py @@ -29,10 +29,12 @@ from django.core.files.uploadedfile import UploadedFile from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db import transaction, connection +from django.db.models import Max from django.http import JsonResponse -from rest_framework import viewsets +from rest_framework import status, viewsets +from rest_framework.response import Response -from api.models import SpacetimeVolume +from api.models import SpacetimeVolume, HistoricalSpacetimeVolume from api.serializers import SpacetimeVolumeSerializer @@ -197,7 +199,7 @@ def _overlaps_queryset(geom, start_date, end_date): ) -def _subtract_geometry(request, overlaps, geom): +def _subtract_geometry(request, overlaps, geom, gid): for entity, stvs in overlaps["db"].items(): overlaps[ "keep" if str(entity) not in request.POST.getlist("overlaps") else "modify" @@ -212,6 +214,7 @@ def _subtract_geometry(request, overlaps, geom): for overlap in overlaps["modify"]: overlap.territory = geom_difference(overlap.territory, geom) + overlap.group = gid if calculate_area(overlap.territory) < AREA_TOLERANCE: overlap.delete() else: @@ -238,6 +241,12 @@ def create(self, request, *args, **kwargs): Solve overlaps if included in request body """ + # Get next historical group id + gid_max = HistoricalSpacetimeVolume.objects.all().aggregate(Max("group"))[ + "group__max" + ] + gid_next = gid_max + 1 if gid_max else 1 + # Validate other data before overlaps empty_territory_data = request.data.copy() empty_territory_data["territory"] = "POINT (0 0)" @@ -277,9 +286,29 @@ def _overlaps(): if "overlaps" not in request.data and len(overlaps["db"]) > 0: return JsonResponse({"overlaps": overlaps["db"]}, status=409) - request.data["territory"] = _subtract_geometry(request, overlaps, geom) + request.data["territory"] = _subtract_geometry( + request, overlaps, geom, gid_next + ) - return super().create(request, *args, **kwargs) + # Group new record + new_serializer = self.get_serializer(data=request.data) + new_serializer.is_valid(raise_exception=True) + data = new_serializer.validated_data.copy() + events = data.pop("related_events", []) + + new_instance = SpacetimeVolume(**data) + if len(overlaps["db"]) > 0: + new_instance.group = gid_next + new_instance.save() + new_instance.related_events.set(events) + new_instance.save_without_historical_record() + + # Return response + return Response( + serializer.data, + status=status.HTTP_201_CREATED, + headers=self.get_success_headers(serializer.data), + ) def update(self, request, *args, **kwargs): """ diff --git a/project/api/views/views.py b/project/api/views/views.py index 1d5a086e..22b5b161 100644 --- a/project/api/views/views.py +++ b/project/api/views/views.py @@ -18,9 +18,11 @@ """ from django.db.models import Count +from django.http import JsonResponse from rest_framework import viewsets, status from rest_framework.response import Response from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.pagination import LimitOffsetPagination from api.models import ( MapColorScheme, @@ -35,7 +37,9 @@ Profile, Symbol, SymbolFeature, -) + HistoricalSpacetimeVolume, + HistoricalTerritorialEntity, +) # pylint: disable=E0611 from api.serializers import ( MapColorSchemeSerializer, TerritorialEntitySerializer, @@ -49,6 +53,9 @@ ProfileSerializer, SymbolSerializer, SymbolFeatureSerializer, + StvHistoryListSerializer, + StvHistoryRetrieveSerializer, + TeHistorySerializer, ) from api.permissions import IsUserOrReadOnly @@ -214,3 +221,85 @@ class ProfileViewSet(viewsets.ModelViewSet): queryset = Profile.objects.all() serializer_class = ProfileSerializer permission_classes = (IsAuthenticatedOrReadOnly, IsUserOrReadOnly) + + +class HistoryPagination(LimitOffsetPagination): + """ + Pagination for history items + """ + + page_size_query_param = "limit" + max_page_size = 1000 + + +class StvHistoryViewSet(viewsets.ReadOnlyModelViewSet): + """ + ViewSet for SpacetimeVolume History + """ + + queryset = HistoricalSpacetimeVolume.objects.all() + pagination_class = HistoryPagination + + def get_queryset(self): + queryset = self.queryset + if self.action == "list": + stv = self.request.query_params.get("stv", None) + if stv is not None: + queryset = queryset.filter(id=stv) + entity = self.request.query_params.get("entity", None) + if entity is not None: + queryset = queryset.filter(entity=entity) + user = self.request.query_params.get("user", None) + if user is not None: + queryset = queryset.filter(history_user=user) + + return queryset + + def get_serializer_class(self): + if self.action == "list": + return StvHistoryListSerializer + if self.action == "retrieve": + return StvHistoryRetrieveSerializer + return StvHistoryListSerializer + + def update(self, request, pk=None): # pylint: disable=R0201 + """ + Reverts model to a certain HistoricalRecord on PUT + """ + HistoricalSpacetimeVolume.objects.get(history_id=pk).instance.save() + return JsonResponse({"status": "Model reverted successfully"}) + + +class TeHistoryViewSet(viewsets.ReadOnlyModelViewSet): + """ + ViewSet for SpacetimeVolume History + """ + + queryset = HistoricalTerritorialEntity.objects.all() + pagination_class = HistoryPagination + serializer_class = TeHistorySerializer + + def get_queryset(self): + + queryset = self.queryset + + if self.action == "list": + + entity = self.request.query_params.get("entity", None) + + if entity is not None: + queryset = queryset.filter(id=entity) + + user = self.request.query_params.get("user", None) + + if user is not None: + queryset = queryset.filter(history_user=user) + + return queryset + + def update(self, request, pk=None): # pylint: disable=R0201 + """ + Reverts model to a certain HistoricalRecord on PUT + """ + HistoricalTerritorialEntity.objects.get(history_id=pk).instance.save() + return JsonResponse({"status": "Model reverted successfully"}) diff --git a/project/chron/settings.py b/project/chron/settings.py index 3705dd82..e91c8c1e 100644 --- a/project/chron/settings.py +++ b/project/chron/settings.py @@ -60,6 +60,7 @@ "drf_firebase_auth", "ordered_model", "silk", + "simple_history", ] if DEBUG: @@ -77,6 +78,7 @@ # "django.contrib.auth.middleware.RemoteUserMiddleware", # "django.middleware.clickjacking.XFrameOptionsMiddleware", "chron.middleware.ErrorMessageFormatter", + "simple_history.middleware.HistoryRequestMiddleware", ] if DEBUG: