Multi-link options for group.get()

Author: mattjala

h5pyd/_hl/group.py

@@ -679,7 +679,19 @@ def __getitem__(self, name, track_order=False):
                 tgt._name = name
         return tgt
 
-    def get(self, name, default=None, getclass=False, getlink=False, track_order=False):
+    def objectify_link_json(self, link_json):
+        if "id" in link_json:
+            link_obj = HardLink(link_json["id"])
+        elif "h5path" in link_json and "h5domain" not in link_json:
+            link_obj = SoftLink(link_json["h5path"])
+        elif "h5path" in link_json and "h5domain" in link_json:
+            link_obj = ExternalLink(link_json["h5domain"], link_json["h5path"])
+        else:
+            raise ValueError("Invalid link JSON")
+
+        return link_obj
+
+    def get(self, name, default=None, getclass=False, getlink=False, track_order=False, **kwds):
         """ Retrieve an item or other information.
 
         "name" given only:
@@ -697,6 +709,21 @@ def get(self, name, default=None, getclass=False, getlink=False, track_order=Fal
             Return HardLink, SoftLink and ExternalLink classes.  Return
             "default" if nothing with that name exists.
 
+        "limit" is an integer:
+            If "name" is None, this will return the first "limit" links in the group.
+
+        "marker" is a string:
+            If "name" is None, this will return only the links that come after the marker in the group's link ordering.
+
+        "pattern" is a string:
+            If "name" is None, this will return only the links that match the given pattern
+            in the target group (and subgroups, if follow_links is provided).
+            Matching is done according to Unix pathname expansion rules.
+
+        "follow_links" is True:
+            If "name" is None, subgroups of the target group will be recursively searched
+            for links that match the given names or pattern.
+
         Example:
 
         >>> cls = group.get('foo', getclass=True)
@@ -709,7 +736,7 @@ def get(self, name, default=None, getclass=False, getlink=False, track_order=Fal
             except KeyError:
                 return default
 
-        if name not in self:
+        if name is not None and name not in self:
             return default
 
         elif getclass and not getlink:
@@ -726,6 +753,52 @@ def get(self, name, default=None, getclass=False, getlink=False, track_order=Fal
                 raise TypeError("Unknown object type")
 
         elif getlink:
+            if name is None:
+                # Get all links in target group(s)
+                # Retrieve "limit", "marker", and "pattern" from kwds
+                limit = kwds.get("limit", None)
+                marker = kwds.get("marker", None)
+                pattern = kwds.get("pattern", None)
+                follow_links = kwds.get("follow_links", False)
+
+                req = "/groups/" + self.id.uuid + "/links"
+                params = {}
+
+                if limit:
+                    params["Limit"] = limit
+                if marker:
+                    params["Marker"] = marker
+                if pattern:
+                    params["pattern"] = pattern
+                if follow_links:
+                    params["follow_links"] = 1
+                if track_order:
+                    params["CreateOrder"] = 1
+
+                rsp = self.GET(req, params=params)
+
+                if "links" in rsp:
+                    # Process list of link objects so they may be accessed by name
+                    links = rsp['links']
+                    links_out = {}
+                    if all([isUUID(k) for k in links]):
+                        # Multiple groups queried, links are returned under group ids
+                        for group_id in links:
+                            group_links = {}
+
+                            for link in links[group_id]:
+                                group_links[link["title"]] = self.objectify_link_json(link)
+
+                            links_out[group_id] = group_links
+
+                    else:
+                        for link in links:
+                            links_out[link["title"]] = self.objectify_link_json(link)
+                else:
+                    raise ValueError("Can't parse server response to links query")
+
+                return links_out
+
             parent_uuid, link_json = self._get_link_json(name)
             typecode = link_json['class']
 
@@ -740,7 +813,7 @@ def get(self, name, default=None, getclass=False, getlink=False, track_order=Fal
 
                 return ExternalLink(link_json['h5domain'], link_json['h5path'])
             elif typecode == 'H5L_TYPE_HARD':
-                return HardLink if getclass else HardLink()
+                return HardLink if getclass else HardLink(link_json['id'])
             else:
                 raise TypeError("Unknown link type")
 
@@ -1214,8 +1287,16 @@ class HardLink(object):
         Represents a hard link in an HDF5 file.  Provided only so that
         Group.get works in a sensible way.  Has no other function.
     """
+    @property
+    # The uuid of the target object
+    def id(self):
+        return self._id
 
-    pass
+    def __init__(self, id=None):
+        self._id = id
+
+    def __repr__(self):
+        return f'<HardLink to "{self.id}">'
 
 
 # TODO: implement equality testing for these

h5pyd/_hl/httpconn.py

@@ -437,6 +437,8 @@ def GET(self, req, format="json", params=None, headers=None, use_cache=True):
         check_cache = self._cache is not None and use_cache and format == "json"
         check_cache = check_cache and params["domain"] == self._domain
         check_cache = check_cache and "select" not in params and "query" not in params
+        check_cache = check_cache and "follow_links" not in params and "pattern" not in params
+        check_cache = check_cache and "Limit" not in params and "Marker" not in params
 
         if check_cache:
             self.log.debug("httpcon - checking cache")
@@ -448,6 +450,7 @@ def GET(self, req, format="json", params=None, headers=None, use_cache=True):
         self.log.info(
             f"GET: {self._endpoint + req} [{params['domain']}] timeout: {self._timeout}"
         )
+
         for k in params:
             if k != "domain":
                 v = params[k]
@@ -462,6 +465,7 @@ def GET(self, req, format="json", params=None, headers=None, use_cache=True):
                 stream = False
             else:
                 stream = True
+
             rsp = s.get(
                 self._endpoint + req,
                 params=params,
@@ -497,6 +501,8 @@ def GET(self, req, format="json", params=None, headers=None, use_cache=True):
 
             add_to_cache = content_type and content_type.startswith("application/json")
             add_to_cache = add_to_cache and content_length < MAX_CACHE_ITEM_SIZE and not req.endswith("/value")
+            add_to_cache = add_to_cache and "follow_links" not in params and "pattern" not in params
+            add_to_cache = add_to_cache and "Limit" not in params and "Marker" not in params
 
             if add_to_cache:
                 # add to our _cache

test/hl/test_group.py

@@ -417,6 +417,110 @@ def test_link_multi_create(self):
             self.assertEqual(link.path, links[i % num_links]._path)
             self.assertEqual(link.filename, links[i % num_links]._filename)
 
+    def test_link_get_multi(self):
+        if config.get("use_h5py"):
+            return
+
+        filename = self.getFileName("test_link_get_multi")
+        print(filename)
+
+        f = h5py.File(filename, 'w')
+        g1 = f.create_group("g1")
+
+        # Create subgroups
+        g2 = g1.create_group("g2")
+        g3 = g2.create_group("g3")
+
+        # Create links in each group
+
+        num_links = 20
+        names = ["link" + str(i) for i in range(num_links)]
+
+        for name in names:
+            g1[name] = g1
+            g2[name] = g2
+            g3[name] = g3
+
+        # Get all links from g1 only
+        links_out = g1.get(None, getlink=True)
+
+        self.assertEqual(len(links_out), num_links + 1)
+
+        for name in names:
+            self.assertTrue(name in links_out)
+            link = links_out[name]
+            self.assertEqual(link.id, g1.id.uuid)
+
+        # Get all links from g1 and subgroups
+        links_out = g1.get(None, getlink=True, follow_links=True)
+
+        # 3 groups containing links
+        self.assertEqual(len(links_out), 3)
+
+        for group_id in [g1.id.uuid, g2.id.uuid, g3.id.uuid]:
+            self.assertTrue(group_id in links_out)
+            links = links_out[group_id]
+
+            if group_id == g3.id.uuid:
+                self.assertEqual(len(links), num_links)
+            else:
+                self.assertEqual(len(links), num_links + 1)
+
+            for name in names:
+                self.assertTrue(name in links)
+                link = links[name]
+                self.assertEqual(link.id, group_id)
+
+        # Make sure cache does not erroneously return recursive links
+        links_out = g1.get(None, getlink=True)
+        self.assertEqual(len(links_out), num_links + 1)
+
+        # Return only 5 links from group
+
+        links_out = g1.get(None, getlink=True, limit=5)
+        self.assertEqual(len(links_out), 5)
+
+        self.assertTrue("g2" in links_out)
+        for name in sorted(names)[0:4]:
+            self.assertTrue(name in links_out)
+            link = links_out[name]
+            self.assertEqual(link.id, g1.id.uuid)
+
+        # Return next 5 links via marker
+        links_out = g1.get(None, getlink=True, limit=5, marker=sorted(names)[3])
+
+        self.assertEqual(len(links_out), 5)
+
+        for name in sorted(names)[4:9]:
+            self.assertTrue(name in links_out)
+            link = links_out[name]
+            self.assertEqual(link.id, g1.id.uuid)
+
+        # Return all links in g1 besides g2
+        links_out = g1.get(None, getlink=True, pattern="link*")
+        self.assertEqual(len(links_out), 20)
+
+        for name in names:
+            if name.startswith("link1"):
+                self.assertTrue(name in links_out)
+                link = links_out[name]
+                self.assertEqual(link.id, g1.id.uuid)
+
+        # Return all links in g1/g2/g3 except for the group links
+        links_out = g1.get(None, getlink=True, follow_links=True, pattern="link*")
+        self.assertEqual(len(links_out), 3)
+
+        for group_id in [g1.id.uuid, g2.id.uuid, g3.id.uuid]:
+            self.assertTrue(group_id in links_out)
+            links = links_out[group_id]
+
+            self.assertEqual(len(links), num_links)
+
+            for name in names:
+                self.assertTrue(name in links)
+                link = links[name]
+                self.assertEqual(link.id, group_id)
+
 
 class TestTrackOrder(TestCase):