WIP: New HTTPConn for each obj in file

Author: mattjala

h5pyd/_hl/dataset.py

@@ -1789,8 +1789,6 @@ def __getitem__(self, args):
                 if next_port > high_port:
                     next_port = low_port
 
-        # TODO: Handle the case where some or all datasets share an HTTPConn object
-
         with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
             # Unwrap one-selection list
             if (isinstance(args, list) and len(args) == 1):
@@ -1843,8 +1841,6 @@ def __setitem__(self, args, vals):
             self.log.debug(msg)
             num_endpoints = 1
 
-        # TODO: Handle the case where some or all datasets share an HTTPConn object
-        # For now, assume each connection is distinct
         if (num_endpoints > 1):
             next_port = low_port
             port_len = len(ports[0])

h5pyd/_hl/group.py

@@ -214,7 +214,6 @@ def create_group(self, h5path):
             self.log.debug("create_group - iterate for link: {}".format(link))
             create_group = False
             req = "/groups/" + parent_uuid + "/links/" + link
-
             try:
                 rsp_json = self.GET(req)
             except IOError as ioe:

h5pyd/_hl/httpconn.py

@@ -323,6 +323,29 @@ def __init__(
             else:
                 self.log.error("Unknown openid provider: {}".format(provider))
 
+    def __copy__(self):
+        new_conn = HttpConn(
+            self._domain,
+            endpoint=self._endpoint,
+            username=self._username,
+            password=self._password,
+            bucket=self._bucket,
+            api_key=self._api_key,
+            mode=self._mode,
+            use_session=self._use_session,
+            use_cache=True if self._cache is not None else False,
+            logger=self._logger,
+            retries=self._retries,
+            timeout=self._timeout,
+        )
+
+        # Share object cache between connections on the same file/domain
+        if self._cache is not None:
+            new_conn._cache = self._cache
+            new_conn._objdb = self._objdb
+
+        return new_conn
+
     def __del__(self):
         if self._hsds:
             self.log.debug("hsds stop")

h5pyd/_hl/objectid.py

@@ -14,7 +14,9 @@
 from datetime import datetime
 import pytz
 import time
+from .httpconn import HttpConn
 from .h5type import createDataType
+from copy import copy
 
 
 def parse_lastmodified(datestr):
@@ -34,7 +36,7 @@ def parse_lastmodified(datestr):
 class ObjectID:
 
     """
-        Uniquely identifies an h5serv resource
+        Uniquely identifies an HSDS resource
     """
 
     @property
@@ -103,9 +105,10 @@ def __init__(self, parent, item, objtype_code=None,
         self._obj_json = item
 
         if http_conn is not None:
-            self._http_conn = http_conn
+            self._http_conn = copy(http_conn)
         elif parent_id is not None and parent_id.http_conn is not None:
-            self._http_conn = parent_id.http_conn
+            # Create new connection with same parameters as parent
+            self._http_conn = copy(parent_id.http_conn)
         else:
             raise IOError("Expected parent to have http connector")
 

test/hl/test_file.py

@@ -118,13 +118,12 @@ def test_create(self):
         self.assertEqual(f.mode, 'r+')
         self.assertEqual(len(f.keys()), 0)
         root_grp = f['/']
-        # f.create_group("subgrp")
         root_grp.create_group("subgrp")
         self.assertEqual(len(f.keys()), 1)
         f.close()
         self.assertEqual(f.id.id, 0)
 
-        # rre-open in append mode
+        # re-open in append mode
         f = h5py.File(filename, "a")
         f.create_group("foo")
         del f["foo"]
@@ -135,6 +134,7 @@ def test_create(self):
             wait_time = 90  # change to >90 to test async updates
             print("waiting {wait_time:d} seconds for root scan sync".format(wait_time=wait_time))
             time.sleep(wait_time)  # let async process update obj number
+
         f = h5py.File(filename, 'r')
         self.assertEqual(f.filename, filename)
         self.assertEqual(f.name, "/")