Fix test_dataset.py, add to testall

Author: mattjala

h5pyd/_hl/dataset.py

@@ -1369,25 +1369,29 @@ def __setitem__(self, args, val):
         # Generally we try to avoid converting the arrays on the Python
         # side.  However, for compound literals this is unavoidable.
         # For h5pyd, do extra check and convert type on client side for efficiency
-        vlen = check_dtype(vlen=self.dtype)
-
-        if not isinstance(val, numpy.ndarray) and vlen is not None and vlen not in (bytes, str):
+        vlen_base_class = check_dtype(vlen=self.dtype)
+        if vlen_base_class is not None and vlen_base_class not in (bytes, str):
             try:
-                val = numpy.asarray(val, dtype=vlen)
+                # Attempt to directly convert the input array of vlen data to its base class
+                val = numpy.asarray(val, dtype=vlen_base_class)
 
             except ValueError as ve:
+                # Failed to convert input array to vlen base class directly, instead create a new array where
+                # each element is an array of the Dataset's dtype
                 self.log.debug(f"asarray ValueError: {ve}")
                 try:
-                    val = numpy.array(
-                        [numpy.array(x, dtype=self.dtype) for x in val],
-                        dtype=self.dtype,
-                    )
+                    # Force output shape
+                    tmp = numpy.empty(shape=val.shape, dtype=self.dtype)
+                    tmp[:] = [numpy.array(x, dtype=self.dtype) for x in val]
+                    val = tmp
                 except ValueError as e:
                     msg = f"ValueError converting value element by element: {e}"
                     self.log.debug(msg)
 
-            if vlen == val.dtype:
+            if vlen_base_class == val.dtype:
                 if val.ndim > 1:
+                    # Reshape array to 2D, where first dim = product of all dims except last, and second dim = last dim
+                    # Then flatten it to 1D
                     tmp = numpy.empty(shape=val.shape[:-1], dtype=self.dtype)
                     tmp.ravel()[:] = [
                         i
@@ -1434,6 +1438,7 @@ def __setitem__(self, args, val):
             else:
                 dtype = self.dtype
                 cast_compound = False
+
             val = numpy.asarray(val, dtype=dtype, order="C")
             if cast_compound:
                 val = val.astype(numpy.dtype([(names[0], dtype)]))
@@ -1520,7 +1525,7 @@ def __setitem__(self, args, val):
             if self.id.uuid.startswith("d-"):
                 # server is HSDS, use binary data, use param values for selection
                 format = "binary"
-                body = arrayToBytes(val, vlen=vlen)
+                body = arrayToBytes(val, vlen=vlen_base_class)
                 self.log.debug(f"writing binary data, {len(body)}")
             else:
                 # h5serv, base64 encode, body json for selection

test/hl/test_vlentype.py

@@ -349,14 +349,16 @@ def test_variable_len_float_dset(self):
         e1 = np.array([1.9, 2.8, 3.7], dtype=np.float64)
 
         data = np.array([e0, e1], dtype=dtvlen)
-        try:
-            # This will fail on HSDS because data is a ndarray of shape (2,3) of floats
+
+        if isinstance(dset.id.id, str):
+            # id is str for HSDS, int for h5py
             dset[...] = data
-            if isinstance(dset.id.id, str):
-                # id is str for HSDS, int for h5py
-                self.assertTrue(False)
-        except ValueError:
-            pass  # expected
+        else:
+            try:
+                # This will fail on h5py due to a different in internal array handling.
+                dset[...] = data
+            except ValueError:
+                pass  # expected on h5py
 
         data = np.zeros((2,), dtype=dtvlen)
         data[0] = e0

testall.py

@@ -18,6 +18,7 @@
 hl_tests = ('test_attribute',
             'test_committedtype',
             'test_complex_numbers',
+            'test_dataset',
             'test_dataset_compound',
             'test_dataset_create',
             'test_dataset_extend',