Add unittest for ray text dedup

tests/tools/test_process_data.py

@@ -164,6 +164,48 @@ def test_ray_image(self):
         for item in dataset['jsonl']:
             self.assertIn('aspect_ratios', item['__dj__stats__'])
 
+    def test_ray_precise_dedup(self):
+        tmp_yaml_file = osp.join(self.tmp_dir, 'config_1.yaml')
+        tmp_out_path = osp.join(self.tmp_dir, 'output_dedup')
+        text_keys = 'text'
+
+        data_path = osp.join(osp.dirname(osp.dirname(osp.dirname(osp.realpath(__file__)))),
+            'demos', 'data', 'demo-dataset-deduplication.jsonl')
+        yaml_config = {
+            'dataset_path': data_path,
+            'executor_type': 'ray',
+            'ray_address': 'auto',
+            'text_keys': text_keys,
+            'image_key': 'images',
+            'export_path': tmp_out_path,
+            'process': [
+                {
+                    'ray_document_deduplicator': {
+                        'backend': 'ray_actor',
+                    },
+                }
+            ]
+        }
+
+        with open(tmp_yaml_file, 'w') as file:
+            yaml.dump(yaml_config, file)
+
+        run_in_subprocess(f'python tools/process_data.py --config {tmp_yaml_file}')
+
+        self.assertTrue(osp.exists(tmp_out_path))
+
+        jsonl_files = [os.path.join(tmp_out_path, f) \
+                       for f in os.listdir(tmp_out_path) \
+                        if f.endswith('.json')]
+        data_cnt = 0
+        for file in jsonl_files:
+            with open(file, 'r') as f:
+                for line in f.readlines():
+                    if line.strip():
+                        data_cnt += 1
+
+        self.assertEqual(data_cnt, 13)
+
 
 if __name__ == '__main__':
     unittest.main()