[Feature]: Add DynamicTable column with reference data

[stephprince]

What would you like to see added to HDMF?

In NeurodataWithoutBorders/pynwb#2055, a user was interested in using "virtual" electrodes that combine multiple existing electrodes in an electrodes table. This use case can be partially addressed by adding rows to the electrodes table for these "virtual" electrodes and adding a ragged column that lists the indices of the electrodes being combined into each "virtual" electrode. However, it would be even better if this additional column could be a column of references to the rows of the electrodes table being combined.

When I attempt to add a column of DynamicTableRegions with the following code, I get an error.

import numpy as np

from datetime import datetime
from uuid import uuid4
from dateutil import tz

from pynwb import NWBHDF5IO, NWBFile, H5DataIO
from pynwb.ecephys import LFP, ElectricalSeries

# setup nwbfile
nwbfile = NWBFile(
    session_description="Mouse exploring an open field",
    identifier=str(uuid4()),
    session_start_time=datetime(2025, 3, 7, 3, 30, 3, tzinfo=tz.gettz("US/Pacific")),
)

# create device
device = nwbfile.create_device(name="array", description="an array", manufacturer="company")

# create an electrode group
electrode_group = nwbfile.create_electrode_group(
    name="shank0",
    description="electrode group for shank 0",
    device=device,
    location="brain area",
)

# add electrodes to the electrode table
n_electrodes = 4
for ch_ix in range(n_electrodes):
    nwbfile.add_electrode(
        group=electrode_group,
        location=f"brain area",
    )
nwbfile.electrodes.location.data[:] = ["brain_area_a", "brain_area_a", "brain_area_b", "brain_area_b"]

 # add virtual electrodes for each brain area
nwbfile.add_electrode_column(name="linked_electrodes", 
                             description="electrodes averaged together for LFP from each brain area", 
                             data=[[]]*4, index=True)
location_values = np.array(nwbfile.electrodes['location'][:])
for i, loc in enumerate(np.unique(location_values)):
    linked_electrodes = np.where(location_values == loc)[0].tolist()
    linked_electrode_region = nwbfile.create_electrode_table_region(region=linked_electrodes, description=f"electrodes in {loc}")
    nwbfile.add_electrode(group=nwbfile.electrode_groups["shank0"], 
                            location=loc, 
                            linked_electrodes=linked_electrode_region)
        
 # create an electrical series of LFP
virtual_electrodes_region = nwbfile.create_electrode_table_region(region=[4, 5], description="virtual LFP channels")
lfp_electrical_series = ElectricalSeries(
    name="series1",
    description="LFP data",
    data=np.random.randn(50, 2),
    electrodes=virtual_electrodes_region,  # reference the virtual electrodes
    starting_time=0.0,
    rate=200.0,
)
lfp = LFP(electrical_series=lfp_electrical_series)
ecephys_module = nwbfile.create_processing_module(
    name="ecephys", description="processed extracellular electrophysiology data"
)
ecephys_module.add(lfp)
nwbfile.add_acquisition(lfp_electrical_series)

# write initial file
with NWBHDF5IO("test_modify_electrodes_table.nwb", "w") as io:
    io.write(nwbfile)

Returns:

Traceback (most recent call last):
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 1439, in __list_fill__
    dset = parent.create_dataset(name, shape=data_shape, dtype=dtype, **io_settings)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/anaconda3/envs/dev/lib/python3.12/site-packages/h5py/_hl/group.py", line 183, in create_dataset
    dsid = dataset.make_new_dset(group, shape, dtype, data, name, **kwds)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/anaconda3/envs/dev/lib/python3.12/site-packages/h5py/_hl/dataset.py", line 87, in make_new_dset
    tid = h5t.py_create(dtype, logical=1)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "h5py/h5t.pyx", line 1669, in h5py.h5t.py_create
  File "h5py/h5t.pyx", line 1693, in h5py.h5t.py_create
  File "h5py/h5t.pyx", line 1753, in h5py.h5t.py_create
TypeError: Object dtype dtype('O') has no native HDF5 equivalent

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/smprince/anaconda3/envs/dev/lib/python3.12/runpy.py", line 198, in _run_module_as_main
    return _run_code(code, main_globals, None,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/anaconda3/envs/dev/lib/python3.12/runpy.py", line 88, in _run_code
    exec(code, run_globals)
  File "/Users/smprince/.vscode/extensions/ms-python.debugpy-2025.4.1/bundled/libs/debugpy/adapter/../../debugpy/launcher/../../debugpy/__main__.py", line 71, in <module>
    cli.main()
  File "/Users/smprince/.vscode/extensions/ms-python.debugpy-2025.4.1/bundled/libs/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 501, in main
    run()
  File "/Users/smprince/.vscode/extensions/ms-python.debugpy-2025.4.1/bundled/libs/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 351, in run_file
    runpy.run_path(target, run_name="__main__")
  File "/Users/smprince/.vscode/extensions/ms-python.debugpy-2025.4.1/bundled/libs/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 310, in run_path
    return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=fname)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/.vscode/extensions/ms-python.debugpy-2025.4.1/bundled/libs/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 127, in _run_module_code
    _run_code(code, mod_globals, init_globals, mod_name, mod_spec, pkg_name, script_name)
  File "/Users/smprince/.vscode/extensions/ms-python.debugpy-2025.4.1/bundled/libs/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 118, in _run_code
    exec(code, run_globals)
  File "/Users/smprince/Documents/code/issue_scripts/pynwb_2055.py", line 105, in <module>
    io.write(nwbfile)
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 395, in write
    super().write(**kwargs)
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/io.py", line 99, in write
    self.write_builder(f_builder, **kwargs)
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 834, in write_builder
    self.write_group(self.__file, gbldr, **kwargs)
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 1013, in write_group
    self.write_group(group, sub_builder, **kwargs)
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 1013, in write_group
    self.write_group(group, sub_builder, **kwargs)
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 1018, in write_group
    self.write_dataset(group, sub_builder, **kwargs)
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 1286, in write_dataset
    dset = self.__list_fill__(parent, name, data, options)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/smprince/Documents/code/hdmf/src/hdmf/backends/hdf5/h5tools.py", line 1443, in __list_fill__
    raise Exception(msg) from exc
Exception: Could not create dataset linked_electrodes in /general/extracellular_ephys/electrodes with shape (4, 1), dtype <class 'pandas.core.frame.DataFrame'>, and iosettings {}. Object dtype dtype('O') has no native HDF5 equivalent

What solution would you like?

Is there a workaround or way to define the column data type as object references?

Do you have any interest in helping implement the feature?

Yes.