Udpate purge tasks (#3167)

*  Remove dataservice link a to a dataset when purging dataset
*  Remove harvest items related to a dataservice when purging it

CHANGELOG.md

@@ -6,6 +6,7 @@
 - Fix the `parse-url` command [#3225](https://github.com/opendatateam/udata/pull/3225)
 - Add `with_drafts` argument to posts API [#3229](https://github.com/opendatateam/udata/pull/3229)
 - Fix failing dataset save in update reuses metrics [#3230](https://github.com/opendatateam/udata/pull/3230)
+- Update purge tasks [#3167](https://github.com/opendatateam/udata/pull/3167)
 
 ## 10.0.5 (2024-12-09)
 

udata/core/dataservices/tasks.py

@@ -1,8 +1,7 @@
 from celery.utils.log import get_task_logger
 
 from udata.core.dataservices.models import Dataservice
-
-# from udata.harvest.models import HarvestJob
+from udata.harvest.models import HarvestJob
 from udata.models import Discussion, Follow, Transfer
 from udata.tasks import job
 
@@ -18,8 +17,7 @@ def purge_dataservices(self):
         # Remove discussions
         Discussion.objects(subject=dataservice).delete()
         # Remove HarvestItem references
-        # TODO: uncomment when adding dataservice harvest
-        # HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
+        HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
         # Remove associated Transfers
         Transfer.objects(subject=dataservice).delete()
         # Remove dataservice

udata/core/dataset/tasks.py

@@ -48,6 +48,11 @@ def purge_datasets(self):
             datasets = topic.datasets
             datasets.remove(dataset)
             topic.update(datasets=datasets)
+        # Remove dataservices related dataset
+        for dataservice in Dataservice.objects(datasets=dataset):
+            datasets = dataservice.datasets
+            datasets.remove(dataset)
+            dataservice.update(datasets=datasets)
         # Remove HarvestItem references
         HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
         # Remove associated Transfers

udata/tests/dataservice/test_dataservice_tasks.py

@@ -3,6 +3,8 @@
 from udata.core.dataservices import tasks
 from udata.core.dataservices.models import Dataservice
 from udata.core.user.factories import UserFactory
+from udata.harvest.models import HarvestItem, HarvestJob
+from udata.harvest.tests.factories import HarvestJobFactory
 from udata.models import Discussion, Follow, Message, Transfer
 from udata.utils import faker
 
@@ -34,9 +36,12 @@ def test_purge_dataservices():
 
     follower = Follow.objects.create(follower=user, following=dataservices[0])
 
+    HarvestJobFactory(items=[HarvestItem(dataservice=dataservices[0])])
+
     tasks.purge_dataservices()
 
     assert Dataservice.objects.count() == 1
     assert Transfer.objects.filter(id=transfer.id).count() == 0
     assert Discussion.objects.filter(id=discussion.id).count() == 0
     assert Follow.objects.filter(id=follower.id).count() == 0
+    assert HarvestJob.objects.filter(items__dataservice=dataservices[0].id).count() == 0

udata/tests/dataset/test_dataset_tasks.py

@@ -1,17 +1,20 @@
 import pytest
 
-from udata.core.dataset import tasks
-
 # Those imports seem mandatory for the csv adapters to be registered.
 # This might be because of the decorator mechanism.
+from udata.core.dataservices.models import Dataservice
+from udata.core.dataset import tasks
 from udata.core.dataset.csv import DatasetCsvAdapter, ResourcesCsvAdapter  # noqa
 from udata.core.dataset.factories import CommunityResourceFactory, DatasetFactory
+from udata.core.discussions.factories import DiscussionFactory
 from udata.core.organization.csv import OrganizationCsvAdapter  # noqa
 from udata.core.reuse.csv import ReuseCsvAdapter  # noqa
 from udata.core.tags.csv import TagCsvAdapter  # noqa
 from udata.core.user.factories import UserFactory
 from udata.harvest.csv import HarvestSourceCsvAdapter  # noqa
-from udata.models import CommunityResource, Dataset, Topic, Transfer
+from udata.harvest.models import HarvestItem, HarvestJob
+from udata.harvest.tests.factories import HarvestJobFactory
+from udata.models import CommunityResource, Dataset, Discussion, Follow, Topic, Transfer
 
 pytestmark = pytest.mark.usefixtures("clean_db")
 
@@ -32,13 +35,26 @@ def test_purge_datasets():
         comment="comment",
     )
 
+    discussion = DiscussionFactory(subject=datasets[0])
+
+    follower = Follow.objects.create(follower=user, following=datasets[0])
+
+    HarvestJobFactory(items=[HarvestItem(dataset=datasets[0])])
+
+    Dataservice.objects.create(title="test", datasets=datasets)
+
     tasks.purge_datasets()
 
     assert Transfer.objects.filter(id=transfer.id).count() == 0
 
     topic = Topic.objects(name="test topic").first()
     assert topic.datasets[0] == datasets[1]
 
+    assert Discussion.objects.filter(id=discussion.id).count() == 0
+    assert Follow.objects.filter(id=follower.id).count() == 0
+    assert HarvestJob.objects.filter(items__dataset=datasets[0].id).count() == 0
+    assert Dataservice.objects.filter(datasets=datasets[0].id).count() == 0
+
 
 def test_purge_datasets_community():
     dataset = Dataset.objects.create(title="delete me", deleted="2016-01-01")