update hdfs

Author: min.du

src/cocozip/io.py

@@ -35,6 +35,14 @@ def imread(filename, flags=cv2.IMREAD_COLOR, zip_file=None):
         path_zip = path[0: pos_at]
         path_img = zip_name + path[pos_at + 1:]
 
+    if path_zip.startswith('hdfs://'):
+        _path_zip = os.path.basename(path_zip)
+        if not os.path.exists(_path_zip):
+            cmd = 'hadoop fs -get %s' % path_zip
+            print(cmd)
+            os.system(cmd)
+        path_zip = _path_zip
+
     if not os.path.isfile(path_zip):
         print("zip file '%s' is not found"%(path_zip))
         assert 0

src/cocozip/torchvision.py

@@ -17,10 +17,6 @@ def __init__(self, root, annFile,
         super(CocoDetection, self).__init__(root, annFile,
                 transform, target_transform, transforms)
         self.zip_file = zip_file
-        if zip_file.startswith('hdfs://'):
-            cmd = 'hdfs fs -get %s' % zip_file
-            os.system(cmd)
-            self.zip_file = os.path.basename(zip_file)
 
     def __getitem__(self, index):
         try:

tests/__pycache__/test_basic.cpython-36.pyc

@@ -26,6 +26,16 @@ def test_imread(self):
             print(img.shape, img2.shape)
             break
 
+    def test_imread_hdfs(self):
+        coco_anno = '/home/public_data/min.du/coco2017/annotations/instances_val2017.json'
+        zip_file = 'hdfs://hobot-bigdata/user/min.du/gpu004/public/coco/zip/val2017.zip'
+        coco = COCO(coco_anno)
+        image_ids = coco.getImgIds()
+        for index in image_ids[:100]:
+            file_name = '%012d.jpg' % index
+            img = cocozip.imread(file_name, zip_file=zip_file)
+        os.system('rm val2017.zip')
+
     def test_imread_speed(self):
         coco_anno = '/home/public_data/min.du/coco2017/annotations/instances_train2017.json'
         imgdir = '/home/public_data/min.du/coco2017/images/train2017'