(add) refresh argument on data saver

matorage/data/saver.py

@@ -80,6 +80,9 @@ class DataSaver(object):
             Keep in mind that using memory is fast because it doesn't use disk IO, but it's not always good.
             If default option(False), then `HDF5_SEC2` driver will be used on posix OS(or `HDF5_WINDOWS` in Windows).
 
+        refresh (:obj:`boolean`, optional, defaults to `False`):
+            All existing data is erased and overwritten.
+
 
     Single Process example
 
@@ -118,6 +121,7 @@ def __init__(
         multipart_upload_size=5 * _MB,
         num_worker_threads=4,
         inmemory=False,
+        refresh=False,
     ):
 
         self.config = config
@@ -147,7 +151,7 @@ def __init__(
             if not check_nas(self.config.endpoint)
             else NAS(self.config.endpoint)
         )
-        self._check_and_create_bucket()
+        self._check_and_create_bucket(refresh=refresh)
 
         self._uploader = Uploader(
             client=self._client,
@@ -191,11 +195,15 @@ def _append_all(self):
                 for name, array in self._datas.items():
                     self._earray[name].append(array[batch_idx, None])
 
-    def _check_and_create_bucket(self):
+    def _check_and_create_bucket(self, refresh):
         if not self._client.bucket_exists(self.config.bucket_name):
             self._client.make_bucket(
                 self.config.bucket_name, location=self.config.region
             )
+        elif refresh:
+            objects = self._client.list_objects(self.config.bucket_name, recursive=True)
+            for obj in objects:
+                self._client.remove_object(self.config.bucket_name, obj.object_name)
 
     def _check_attr_name(self, name):
         """

tests/test_datasaver.py

@@ -381,6 +381,19 @@ def test_datasaver_nas(self):
         self.data_saver({"x": x})
         self.data_saver.disconnect()
 
+    def test_datasaver_refresh(self):
+
+        self.data_config = DataConfig(
+            **self.storage_config,
+            dataset_name="test_datasaver_refresh",
+            attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
+        )
+        for refresh in [False, True]:
+            self.data_saver = DataSaver(config=self.data_config, refresh=refresh)
+            x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
+            self.assertEqual(x.shape, (3, 2))
+            self.data_saver({"x": x})
+            self.data_saver.disconnect()
 
 @unittest.skipIf(
     'access_key' not in os.environ or 'secret_key' not in os.environ, 'S3 Skip'