diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eba8485e..b1283e11 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,10 @@ jobs: nwb_models/pyproject.toml - name: Install dependencies - run: pip install -e .[tests] + run: | + pip install -e .[tests] + pip install -e ../nwb_schema_language + pip install -e ../nwb_models working-directory: nwb_linkml - name: Run Tests diff --git a/docs/meta/todo.md b/docs/meta/todo.md index d2bf9acc..dd9f7502 100644 --- a/docs/meta/todo.md +++ b/docs/meta/todo.md @@ -49,6 +49,10 @@ Remove monkeypatches/overrides once PRs are closed Tests - [ ] Ensure schemas and pydantic modules in repos are up to date +Loading +- [ ] Top-level containers are still a little janky, eg. how `ProcessingModule` just accepts + extra args rather than properly abstracting `value` as a `__getitem__(self, key) -> T:` + ## Docs TODOs ```{todolist} diff --git a/nwb_linkml/conftest.py b/nwb_linkml/conftest.py index 450875f5..88c09a6e 100644 --- a/nwb_linkml/conftest.py +++ b/nwb_linkml/conftest.py @@ -71,7 +71,7 @@ def parse_adapter_blocks(document: Document) -> Generator[Region, None, None]: doctest_parser = Sybil( parsers=[DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), PythonCodeBlockParser()], - patterns=["*.py"], + patterns=["providers/git.py"], ) pytest_collect_file = (adapter_parser + doctest_parser).pytest() diff --git a/nwb_linkml/pdm.lock b/nwb_linkml/pdm.lock index 7a1aca7f..f6f2c7cd 100644 --- a/nwb_linkml/pdm.lock +++ b/nwb_linkml/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "plot", "tests"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:f219083028bd024c53bc55626c8b6088d6eb5c2ade56bd694a7a112098aa9bfc" +content_hash = "sha256:1c297e11f6dc9e4f6b8d29df872177d2ce65bbd334c0b65aa5175dfb125c4d9f" [[metadata.targets]] requires_python = ">=3.10,<3.13" @@ -549,7 +549,7 @@ name = "h5py" version = "3.11.0" requires_python = ">=3.8" summary = "Read and write HDF5 files from Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "numpy>=1.17.3", ] @@ -580,6 +580,26 @@ files = [ {file = "hbreader-0.9.1.tar.gz", hash = "sha256:d2c132f8ba6276d794c66224c3297cec25c8079d0a4cf019c061611e0a3b94fa"}, ] +[[package]] +name = "hdmf" +version = "3.14.3" +requires_python = ">=3.8" +summary = "A hierarchical data modeling framework for modern science data standards" +groups = ["dev", "tests"] +dependencies = [ + "h5py>=2.10", + "importlib-resources; python_version < \"3.9\"", + "jsonschema>=2.6.0", + "numpy>=1.18", + "pandas>=1.0.5", + "ruamel-yaml>=0.16", + "scipy>=1.4", +] +files = [ + {file = "hdmf-3.14.3-py3-none-any.whl", hash = "sha256:1417ccc0d336d535192b7a3db4c7354cbc15123f1ccb3cdd82e363308e78f9bc"}, + {file = "hdmf-3.14.3.tar.gz", hash = "sha256:e9548fc7bdbb534a2750092b6b9819df2ce50e27430866c3c32061a2306271cc"}, +] + [[package]] name = "idna" version = "3.8" @@ -751,7 +771,7 @@ name = "jsonschema" version = "4.23.0" requires_python = ">=3.8" summary = "An implementation of JSON Schema validation for Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "attrs>=22.2.0", "importlib-resources>=1.4.0; python_version < \"3.9\"", @@ -770,7 +790,7 @@ name = "jsonschema-specifications" version = "2023.12.1" requires_python = ">=3.8" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", @@ -976,7 +996,7 @@ name = "networkx" version = "3.3" requires_python = ">=3.10" summary = "Python package for creating and manipulating graphs and networks" -groups = ["dev", "tests"] +groups = ["default", "dev", "tests"] files = [ {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"}, {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"}, @@ -984,45 +1004,36 @@ files = [ [[package]] name = "numpy" -version = "2.1.0" -requires_python = ">=3.10" +version = "1.26.4" +requires_python = ">=3.9" summary = "Fundamental package for array computing in Python" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ - {file = "numpy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:30014b234f07b5fec20f4146f69e13cfb1e33ee9a18a1879a0142fbb00d47673"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:899da829b362ade41e1e7eccad2cf274035e1cb36ba73034946fccd4afd8606b"}, - {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08801848a40aea24ce16c2ecde3b756f9ad756586fb2d13210939eb69b023f5b"}, - {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:398049e237d1aae53d82a416dade04defed1a47f87d18d5bd615b6e7d7e41d1f"}, - {file = "numpy-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0abb3916a35d9090088a748636b2c06dc9a6542f99cd476979fb156a18192b84"}, - {file = "numpy-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10e2350aea18d04832319aac0f887d5fcec1b36abd485d14f173e3e900b83e33"}, - {file = "numpy-2.1.0-cp310-cp310-win32.whl", hash = "sha256:f6b26e6c3b98adb648243670fddc8cab6ae17473f9dc58c51574af3e64d61211"}, - {file = "numpy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:f505264735ee074250a9c78247ee8618292091d9d1fcc023290e9ac67e8f1afa"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd"}, - {file = "numpy-2.1.0-cp311-cp311-win32.whl", hash = "sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e"}, - {file = "numpy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fe76d75b345dc045acdbc006adcb197cc680754afd6c259de60d358d60c93736"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f358ea9e47eb3c2d6eba121ab512dfff38a88db719c38d1e67349af210bc7529"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:dd94ce596bda40a9618324547cfaaf6650b1a24f5390350142499aa4e34e53d1"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b47c551c6724960479cefd7353656498b86e7232429e3a41ab83be4da1b109e8"}, - {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0756a179afa766ad7cb6f036de622e8a8f16ffdd55aa31f296c870b5679d745"}, - {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24003ba8ff22ea29a8c306e61d316ac74111cebf942afbf692df65509a05f111"}, - {file = "numpy-2.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b34fa5e3b5d6dc7e0a4243fa0f81367027cb6f4a7215a17852979634b5544ee0"}, - {file = "numpy-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c4f982715e65036c34897eb598d64aef15150c447be2cfc6643ec7a11af06574"}, - {file = "numpy-2.1.0-cp312-cp312-win32.whl", hash = "sha256:c4cd94dfefbefec3f8b544f61286584292d740e6e9d4677769bc76b8f41deb02"}, - {file = "numpy-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0cdef204199278f5c461a0bed6ed2e052998276e6d8ab2963d5b5c39a0500bc"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2"}, - {file = "numpy-2.1.0.tar.gz", hash = "sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] [[package]] @@ -1102,7 +1113,7 @@ name = "pandas" version = "2.2.2" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "numpy>=1.22.4; python_version < \"3.11\"", "numpy>=1.23.2; python_version == \"3.11\"", @@ -1350,6 +1361,24 @@ files = [ {file = "PyJSG-0.11.10.tar.gz", hash = "sha256:4bd6e3ff2833fa2b395bbe803a2d72a5f0bab5b7285bccd0da1a1bc0aee88bfa"}, ] +[[package]] +name = "pynwb" +version = "2.8.1" +requires_python = ">=3.8" +summary = "Package for working with Neurodata stored in the NWB format." +groups = ["dev", "tests"] +dependencies = [ + "h5py>=2.10", + "hdmf>=3.14.0", + "numpy<2.0,>=1.18", + "pandas>=1.1.5", + "python-dateutil>=2.7.3", +] +files = [ + {file = "pynwb-2.8.1-py3-none-any.whl", hash = "sha256:f3c392652b26396e135cf6f1abd570d413c9eb7bf5bdb1a89d899852338fdf6c"}, + {file = "pynwb-2.8.1.tar.gz", hash = "sha256:498e4bc46a7b0a1331a0f754bac72ea7f9d10d1bba35af3c7be78a61bb1d104b"}, +] + [[package]] name = "pyparsing" version = "3.1.4" @@ -1469,7 +1498,7 @@ name = "python-dateutil" version = "2.9.0.post0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Extensions to the standard Python datetime module" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "six>=1.5", ] @@ -1506,7 +1535,7 @@ files = [ name = "pytz" version = "2024.1" summary = "World timezone definitions, modern and historical" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, @@ -1597,7 +1626,7 @@ name = "referencing" version = "0.35.1" requires_python = ">=3.8" summary = "JSON Referencing + Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "attrs>=22.2.0", "rpds-py>=0.7.0", @@ -1701,7 +1730,7 @@ name = "rpds-py" version = "0.20.0" requires_python = ">=3.8" summary = "Python bindings to Rust's persistent data structures (rpds)" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"}, {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"}, @@ -1762,7 +1791,7 @@ name = "ruamel-yaml" version = "0.18.6" requires_python = ">=3.7" summary = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "ruamel-yaml-clib>=0.2.7; platform_python_implementation == \"CPython\" and python_version < \"3.13\"", ] @@ -1776,7 +1805,7 @@ name = "ruamel-yaml-clib" version = "0.2.8" requires_python = ">=3.6" summary = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -groups = ["default"] +groups = ["default", "dev", "tests"] marker = "platform_python_implementation == \"CPython\" and python_version < \"3.13\"" files = [ {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, @@ -1833,6 +1862,43 @@ files = [ {file = "ruff-0.6.2.tar.gz", hash = "sha256:239ee6beb9e91feb8e0ec384204a763f36cb53fb895a1a364618c6abb076b3be"}, ] +[[package]] +name = "scipy" +version = "1.14.1" +requires_python = ">=3.10" +summary = "Fundamental algorithms for scientific computing in Python" +groups = ["dev", "tests"] +dependencies = [ + "numpy<2.3,>=1.23.5", +] +files = [ + {file = "scipy-1.14.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:b28d2ca4add7ac16ae8bb6632a3c86e4b9e4d52d3e34267f6e1b0c1f8d87e389"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d0d2821003174de06b69e58cef2316a6622b60ee613121199cb2852a873f8cf3"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8bddf15838ba768bb5f5083c1ea012d64c9a444e16192762bd858f1e126196d0"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:97c5dddd5932bd2a1a31c927ba5e1463a53b87ca96b5c9bdf5dfd6096e27efc3"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ff0a7e01e422c15739ecd64432743cf7aae2b03f3084288f399affcefe5222d"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e32dced201274bf96899e6491d9ba3e9a5f6b336708656466ad0522d8528f69"}, + {file = "scipy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8426251ad1e4ad903a4514712d2fa8fdd5382c978010d1c6f5f37ef286a713ad"}, + {file = "scipy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:a49f6ed96f83966f576b33a44257d869756df6cf1ef4934f59dd58b25e0327e5"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:2da0469a4ef0ecd3693761acbdc20f2fdeafb69e6819cc081308cc978153c675"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c0ee987efa6737242745f347835da2cc5bb9f1b42996a4d97d5c7ff7928cb6f2"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3a1b111fac6baec1c1d92f27e76511c9e7218f1695d61b59e05e0fe04dc59617"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8475230e55549ab3f207bff11ebfc91c805dc3463ef62eda3ccf593254524ce8"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:278266012eb69f4a720827bdd2dc54b2271c97d84255b2faaa8f161a158c3b37"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fef8c87f8abfb884dac04e97824b61299880c43f4ce675dd2cbeadd3c9b466d2"}, + {file = "scipy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b05d43735bb2f07d689f56f7b474788a13ed8adc484a85aa65c0fd931cf9ccd2"}, + {file = "scipy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:716e389b694c4bb564b4fc0c51bc84d381735e0d39d3f26ec1af2556ec6aad94"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066"}, + {file = "scipy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1"}, + {file = "scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f"}, + {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, +] + [[package]] name = "setuptools" version = "74.0.0" @@ -2023,7 +2089,7 @@ name = "tzdata" version = "2024.1" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, diff --git a/nwb_linkml/pyproject.toml b/nwb_linkml/pyproject.toml index 41cf80a3..c8ccd36e 100644 --- a/nwb_linkml/pyproject.toml +++ b/nwb_linkml/pyproject.toml @@ -9,7 +9,7 @@ license = {text = "AGPL-3.0"} readme = "README.md" requires-python = "<3.13,>=3.10" dependencies = [ - "nwb-models>=0.1.0", + "nwb-models>=0.2.0", "pyyaml>=6.0", "linkml-runtime>=1.7.7", "nwb-schema-language>=0.1.3", @@ -22,9 +22,10 @@ dependencies = [ "pydantic-settings>=2.0.3", "tqdm>=4.66.1", 'typing-extensions>=4.12.2;python_version<"3.11"', - "numpydantic>=1.3.3", + "numpydantic>=1.5.0", "black>=24.4.2", "pandas>=2.2.2", + "networkx>=3.3", ] [project.urls] @@ -44,6 +45,7 @@ tests = [ "pytest-cov<5.0.0,>=4.1.0", "sybil>=6.0.3", "requests-cache>=1.2.1", + "pynwb>=2.8.1", ] dev = [ "nwb-linkml[tests]", diff --git a/nwb_linkml/src/nwb_linkml/adapters/adapter.py b/nwb_linkml/src/nwb_linkml/adapters/adapter.py index 13e86fdf..acbc8966 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/adapter.py +++ b/nwb_linkml/src/nwb_linkml/adapters/adapter.py @@ -2,6 +2,7 @@ Base class for adapters """ +import os import sys from abc import abstractmethod from dataclasses import dataclass, field @@ -101,6 +102,19 @@ class Adapter(BaseModel): """Abstract base class for adapters""" _logger: Optional[Logger] = None + _debug: Optional[bool] = None + + @property + def debug(self) -> bool: + """ + Whether we are in debug mode, which adds extra metadata in generated elements. + + Set explicitly via ``_debug`` , or else checks for the truthiness of the + environment variable ``NWB_LINKML_DEBUG`` + """ + if self._debug is None: + self._debug = bool(os.environ.get("NWB_LINKML_DEBUG", False)) + return self._debug @property def logger(self) -> Logger: diff --git a/nwb_linkml/src/nwb_linkml/adapters/attribute.py b/nwb_linkml/src/nwb_linkml/adapters/attribute.py index ddf6edb9..7ae2ea1e 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/attribute.py +++ b/nwb_linkml/src/nwb_linkml/adapters/attribute.py @@ -10,7 +10,7 @@ from nwb_linkml.adapters.adapter import Adapter, BuildResult, is_1d from nwb_linkml.adapters.array import ArrayAdapter from nwb_linkml.maps import Map -from nwb_linkml.maps.dtype import handle_dtype +from nwb_linkml.maps.dtype import handle_dtype, inlined from nwb_schema_language import Attribute @@ -104,6 +104,7 @@ def apply(cls, attr: Attribute, res: Optional[BuildResult] = None) -> BuildResul range=handle_dtype(attr.dtype), description=attr.doc, required=attr.required, + inlined=inlined(attr.dtype), **cls.handle_defaults(attr), ) return BuildResult(slots=[slot]) @@ -151,6 +152,7 @@ def apply(cls, attr: Attribute, res: Optional[BuildResult] = None) -> BuildResul multivalued=multivalued, description=attr.doc, required=attr.required, + inlined=inlined(attr.dtype), **expressions, **cls.handle_defaults(attr), ) @@ -171,7 +173,10 @@ def build(self) -> "BuildResult": Build the slot definitions, every attribute should have a map. """ map = self.match() - return map.apply(self.cls) + res = map.apply(self.cls) + if self.debug: # pragma: no cover - only used in development + res = self._amend_debug(res, map) + return res def match(self) -> Optional[Type[AttributeMap]]: """ @@ -195,3 +200,13 @@ def match(self) -> Optional[Type[AttributeMap]]: return None else: return matches[0] + + def _amend_debug( + self, res: BuildResult, map: Optional[Type[AttributeMap]] = None + ) -> BuildResult: # pragma: no cover - only used in development + map_name = "None" if map is None else map.__name__ + for cls in res.classes: + cls.annotations["attribute_map"] = {"tag": "attribute_map", "value": map_name} + for slot in res.slots: + slot.annotations["attribute_map"] = {"tag": "attribute_map", "value": map_name} + return res diff --git a/nwb_linkml/src/nwb_linkml/adapters/classes.py b/nwb_linkml/src/nwb_linkml/adapters/classes.py index 0097e47d..c008f717 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/classes.py +++ b/nwb_linkml/src/nwb_linkml/adapters/classes.py @@ -92,6 +92,13 @@ def build_base(self, extra_attrs: Optional[List[SlotDefinition]] = None) -> Buil # Get vanilla top-level attributes kwargs["attributes"].extend(self.build_attrs(self.cls)) + if self.debug: # pragma: no cover - only used in development + kwargs["annotations"] = {} + kwargs["annotations"]["group_adapter"] = { + "tag": "group_adapter", + "value": "container_slot", + } + if extra_attrs is not None: if isinstance(extra_attrs, SlotDefinition): extra_attrs = [extra_attrs] @@ -230,18 +237,23 @@ def build_name_slot(self) -> SlotDefinition: ifabsent=f"string({name})", equals_string=equals_string, range="string", + identifier=True, ) else: - name_slot = SlotDefinition(name="name", required=True, range="string") + name_slot = SlotDefinition(name="name", required=True, range="string", identifier=True) return name_slot def build_self_slot(self) -> SlotDefinition: """ If we are a child class, we make a slot so our parent can refer to us """ - return SlotDefinition( + slot = SlotDefinition( name=self._get_slot_name(), description=self.cls.doc, range=self._get_full_name(), + inlined=True, **QUANTITY_MAP[self.cls.quantity], ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "self_slot"} + return slot diff --git a/nwb_linkml/src/nwb_linkml/adapters/dataset.py b/nwb_linkml/src/nwb_linkml/adapters/dataset.py index ef5eb61b..f0b0053e 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/dataset.py +++ b/nwb_linkml/src/nwb_linkml/adapters/dataset.py @@ -11,7 +11,7 @@ from nwb_linkml.adapters.array import ArrayAdapter from nwb_linkml.adapters.classes import ClassAdapter from nwb_linkml.maps import QUANTITY_MAP, Map -from nwb_linkml.maps.dtype import flat_to_linkml, handle_dtype +from nwb_linkml.maps.dtype import flat_to_linkml, handle_dtype, inlined from nwb_linkml.maps.naming import camel_to_snake from nwb_schema_language import Dataset @@ -147,6 +147,7 @@ class MapScalarAttributes(DatasetMap): name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -245,6 +246,7 @@ class MapListlike(DatasetMap): attributes: name: name: name + identifier: true range: string required: true value: @@ -257,6 +259,8 @@ class MapListlike(DatasetMap): range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true """ @@ -299,6 +303,8 @@ def apply( description=cls.doc, required=cls.quantity not in ("*", "?"), annotations=[{"source_type": "reference"}], + inlined=True, + inlined_as_list=True, ) res.classes[0].attributes["value"] = slot return res @@ -384,13 +390,11 @@ def check(c, cls: Dataset) -> bool: - ``False`` """ - dtype = handle_dtype(cls.dtype) return ( cls.name and (all([cls.dims, cls.shape]) or cls.neurodata_type_inc == "VectorData") and not has_attrs(cls) and not is_compound(cls) - and dtype in flat_to_linkml ) @classmethod @@ -418,6 +422,7 @@ def apply( range=handle_dtype(cls.dtype), description=cls.doc, required=cls.quantity not in ("*", "?"), + inlined=inlined(cls.dtype), **expressions, ) ] @@ -430,6 +435,10 @@ class MapArrayLikeAttributes(DatasetMap): The most general case - treat everything that isn't handled by one of the special cases as an array! + We specifically include classes that have no attributes but also don't have a name, + as they still require their own class (unlike :class:`.MapArrayLike` above, where we + just generate an anonymous slot.) + Examples: .. adapter:: DatasetAdapter @@ -478,6 +487,7 @@ class MapArrayLikeAttributes(DatasetMap): attributes: name: name: name + identifier: true range: string required: true resolution: @@ -525,7 +535,7 @@ def check(c, cls: Dataset) -> bool: return ( all([cls.dims, cls.shape]) and cls.neurodata_type_inc != "VectorData" - and has_attrs(cls) + and (has_attrs(cls) or not cls.name) and not is_compound(cls) and (dtype == "AnyType" or dtype in flat_to_linkml) ) @@ -540,7 +550,9 @@ def apply( array_adapter = ArrayAdapter(cls.dims, cls.shape) expressions = array_adapter.make_slot() # make a slot for the arraylike class - array_slot = SlotDefinition(name="value", range=handle_dtype(cls.dtype), **expressions) + array_slot = SlotDefinition( + name="value", range=handle_dtype(cls.dtype), inlined=inlined(cls.dtype), **expressions + ) res.classes[0].attributes.update({"value": array_slot}) return res @@ -579,6 +591,7 @@ def apply( description=cls.doc, range=f"{cls.neurodata_type_inc}", annotations=[{"named": True}, {"source_type": "neurodata_type_inc"}], + inlined=True, **QUANTITY_MAP[cls.quantity], ) res = BuildResult(slots=[this_slot]) @@ -590,102 +603,6 @@ def apply( # -------------------------------------------------- -class MapVectorClassRange(DatasetMap): - """ - Map a ``VectorData`` class that is a reference to another class as simply - a multivalued slot range, rather than an independent class - """ - - @classmethod - def check(c, cls: Dataset) -> bool: - """ - Check that we are a VectorData object without any additional attributes - with a dtype that refers to another class - """ - dtype = handle_dtype(cls.dtype) - return ( - cls.neurodata_type_inc == "VectorData" - and cls.name - and not has_attrs(cls) - and not (cls.shape or cls.dims) - and not is_compound(cls) - and dtype not in flat_to_linkml - ) - - @classmethod - def apply( - c, cls: Dataset, res: Optional[BuildResult] = None, name: Optional[str] = None - ) -> BuildResult: - """ - Create a slot that replaces the base class just as a list[ClassRef] - """ - this_slot = SlotDefinition( - name=cls.name, - description=cls.doc, - multivalued=True, - range=handle_dtype(cls.dtype), - required=cls.quantity not in ("*", "?"), - ) - res = BuildResult(slots=[this_slot]) - return res - - -# -# class Map1DVector(DatasetMap): -# """ -# ``VectorData`` is subclassed with a name but without dims or attributes, -# treat this as a normal 1D array slot that replaces any class that would be built for this -# -# eg. all the datasets in epoch.TimeIntervals: -# -# .. code-block:: yaml -# -# groups: -# - neurodata_type_def: TimeIntervals -# neurodata_type_inc: DynamicTable -# doc: A container for aggregating epoch data and the TimeSeries that each epoch applies -# to. -# datasets: -# - name: start_time -# neurodata_type_inc: VectorData -# dtype: float32 -# doc: Start time of epoch, in seconds. -# -# """ -# -# @classmethod -# def check(c, cls: Dataset) -> bool: -# """ -# Check that we're a 1d VectorData class -# """ -# return ( -# cls.neurodata_type_inc == "VectorData" -# and not cls.dims -# and not cls.shape -# and not cls.attributes -# and not cls.neurodata_type_def -# and not is_compound(cls) -# and cls.name -# ) -# -# @classmethod -# def apply( -# c, cls: Dataset, res: Optional[BuildResult] = None, name: Optional[str] = None -# ) -> BuildResult: -# """ -# Return a simple multivalued slot -# """ -# this_slot = SlotDefinition( -# name=cls.name, -# description=cls.doc, -# range=handle_dtype(cls.dtype), -# multivalued=True, -# ) -# # No need to make a class for us, so we replace the existing build results -# res = BuildResult(slots=[this_slot]) -# return res - - class MapNVectors(DatasetMap): """ An unnamed container that indicates an arbitrary quantity of some other neurodata type. @@ -795,6 +712,7 @@ def apply( description=a_dtype.doc, range=handle_dtype(a_dtype.dtype), array=ArrayExpression(exact_number_dimensions=1), + inlined=inlined(a_dtype.dtype), **QUANTITY_MAP[cls.quantity], ) res.classes[0].attributes.update(slots) @@ -826,6 +744,8 @@ def build(self) -> BuildResult: if map is not None: res = map.apply(self.cls, res, self._get_full_name()) + if self.debug: # pragma: no cover - only used in development + res = self._amend_debug(res, map) return res def match(self) -> Optional[Type[DatasetMap]]: @@ -850,3 +770,13 @@ def match(self) -> Optional[Type[DatasetMap]]: return None else: return matches[0] + + def _amend_debug( + self, res: BuildResult, map: Optional[Type[DatasetMap]] = None + ) -> BuildResult: # pragma: no cover - only used in development + map_name = "None" if map is None else map.__name__ + for cls in res.classes: + cls.annotations["dataset_map"] = {"tag": "dataset_map", "value": map_name} + for slot in res.slots: + slot.annotations["dataset_map"] = {"tag": "dataset_map", "value": map_name} + return res diff --git a/nwb_linkml/src/nwb_linkml/adapters/group.py b/nwb_linkml/src/nwb_linkml/adapters/group.py index 13a03b71..0703aa03 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/group.py +++ b/nwb_linkml/src/nwb_linkml/adapters/group.py @@ -68,11 +68,17 @@ def build_links(self) -> List[SlotDefinition]: if not self.cls.links: return [] + annotations = [{"tag": "source_type", "value": "link"}] + + if self.debug: # pragma: no cover - only used in development + annotations.append({"tag": "group_adapter", "value": "link"}) + slots = [ SlotDefinition( name=link.name, any_of=[{"range": link.target_type}, {"range": "string"}], - annotations=[{"tag": "source_type", "value": "link"}], + annotations=annotations, + inlined=True, **QUANTITY_MAP[link.quantity], ) for link in self.cls.links @@ -111,6 +117,9 @@ def handle_container_group(self, cls: Group) -> BuildResult: inlined_as_list=False, ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "container_group"} + if self.parent is not None: # if we have a parent, # just return the slot itself without the class @@ -144,17 +153,20 @@ def handle_container_slot(self, cls: Group) -> BuildResult: """ name = camel_to_snake(self.cls.neurodata_type_inc) if not self.cls.name else cls.name - return BuildResult( - slots=[ - SlotDefinition( - name=name, - range=self.cls.neurodata_type_inc, - description=self.cls.doc, - **QUANTITY_MAP[cls.quantity], - ) - ] + slot = SlotDefinition( + name=name, + range=self.cls.neurodata_type_inc, + description=self.cls.doc, + inlined=True, + inlined_as_list=False, + **QUANTITY_MAP[cls.quantity], ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "container_slot"} + + return BuildResult(slots=[slot]) + def build_subclasses(self) -> BuildResult: """ Build nested groups and datasets @@ -166,20 +178,9 @@ def build_subclasses(self) -> BuildResult: # for creating slots vs. classes is handled by the adapter class dataset_res = BuildResult() for dset in self.cls.datasets: - # if dset.name == 'timestamps': - # pdb.set_trace() dset_adapter = DatasetAdapter(cls=dset, parent=self) dataset_res += dset_adapter.build() - # Actually i'm not sure we have to special case this, we could handle it in - # i/o instead - - # Groups are a bit more complicated because they can also behave like - # range declarations: - # eg. a group can have multiple groups with `neurodata_type_inc`, no name, - # and quantity of *, - # the group can then contain any number of groups of those included types as direct children - group_res = BuildResult() for group in self.cls.groups: @@ -190,6 +191,33 @@ def build_subclasses(self) -> BuildResult: return res + def build_self_slot(self) -> SlotDefinition: + """ + If we are a child class, we make a slot so our parent can refer to us + + Groups are a bit more complicated because they can also behave like + range declarations: + eg. a group can have multiple groups with `neurodata_type_inc`, no name, + and quantity of *, + the group can then contain any number of groups of those included types as direct children + + We make sure that we're inlined as a dict so our parent class can refer to us like:: + + parent.{slot_name}[{name}] = self + + """ + slot = SlotDefinition( + name=self._get_slot_name(), + description=self.cls.doc, + range=self._get_full_name(), + inlined=True, + inlined_as_list=True, + **QUANTITY_MAP[self.cls.quantity], + ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "container_slot"} + return slot + def _check_if_container(self, group: Group) -> bool: """ Check if a given subgroup is a container subgroup, diff --git a/nwb_linkml/src/nwb_linkml/adapters/namespaces.py b/nwb_linkml/src/nwb_linkml/adapters/namespaces.py index 266906e2..c6abd708 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/namespaces.py +++ b/nwb_linkml/src/nwb_linkml/adapters/namespaces.py @@ -48,7 +48,16 @@ def from_yaml(cls, path: Path) -> "NamespacesAdapter": need_imports = [] for needed in ns_adapter.needed_imports.values(): - need_imports.extend([n for n in needed if n not in ns_adapter.needed_imports]) + # try to locate imports implied by the namespace schema, + # but are either not provided by the current namespace + # or are otherwise already provided in `imported` by the loader function + need_imports.extend( + [ + n + for n in needed + if n not in ns_adapter.needed_imports and n not in ns_adapter.versions + ] + ) for needed in need_imports: if needed in DEFAULT_REPOS: @@ -56,6 +65,8 @@ def from_yaml(cls, path: Path) -> "NamespacesAdapter": needed_adapter = NamespacesAdapter.from_yaml(needed_source_ns) ns_adapter.imported.append(needed_adapter) + ns_adapter.populate_imports() + return ns_adapter def build( @@ -176,7 +187,6 @@ def find_type_source(self, name: str) -> SchemaAdapter: else: raise KeyError(f"No schema found that define {name}") - @model_validator(mode="after") def populate_imports(self) -> "NamespacesAdapter": """ Populate the imports that are needed for each schema file diff --git a/nwb_linkml/src/nwb_linkml/generators/pydantic.py b/nwb_linkml/src/nwb_linkml/generators/pydantic.py index 0f824afd..1928cf58 100644 --- a/nwb_linkml/src/nwb_linkml/generators/pydantic.py +++ b/nwb_linkml/src/nwb_linkml/generators/pydantic.py @@ -6,11 +6,10 @@ """ import re -import sys from dataclasses import dataclass, field from pathlib import Path from types import ModuleType -from typing import ClassVar, Dict, List, Optional, Tuple +from typing import Callable, ClassVar, Dict, List, Literal, Optional, Tuple from linkml.generators import PydanticGenerator from linkml.generators.pydanticgen.array import ArrayRepresentation, NumpydanticArray @@ -23,11 +22,14 @@ SlotDefinition, SlotDefinitionName, ) -from linkml_runtime.utils.compile_python import file_text from linkml_runtime.utils.formatutils import remove_empty_items from linkml_runtime.utils.schemaview import SchemaView -from nwb_linkml.includes.base import BASEMODEL_GETITEM +from nwb_linkml.includes.base import ( + BASEMODEL_COERCE_CHILD, + BASEMODEL_COERCE_VALUE, + BASEMODEL_GETITEM, +) from nwb_linkml.includes.hdmf import ( DYNAMIC_TABLE_IMPORTS, DYNAMIC_TABLE_INJECTS, @@ -36,7 +38,7 @@ ) from nwb_linkml.includes.types import ModelTypeString, NamedImports, NamedString, _get_name -OPTIONAL_PATTERN = re.compile(r"Optional\[([\w\.]*)\]") +OPTIONAL_PATTERN = re.compile(r"Optional\[(.*)\]") @dataclass @@ -52,6 +54,8 @@ class NWBPydanticGenerator(PydanticGenerator): ), 'object_id: Optional[str] = Field(None, description="Unique UUID for each object")', BASEMODEL_GETITEM, + BASEMODEL_COERCE_VALUE, + BASEMODEL_COERCE_CHILD, ) split: bool = True imports: list[Import] = field(default_factory=lambda: [Import(module="numpy", alias="np")]) @@ -66,6 +70,7 @@ class NWBPydanticGenerator(PydanticGenerator): emit_metadata: bool = True gen_classvars: bool = True gen_slots: bool = True + extra_fields: Literal["allow", "forbid", "ignore"] = "allow" skip_meta: ClassVar[Tuple[str]] = ("domain_of", "alias") @@ -131,6 +136,8 @@ def after_generate_class(self, cls: ClassResult, sv: SchemaView) -> ClassResult: """Customize dynamictable behavior""" cls = AfterGenerateClass.inject_dynamictable(cls) cls = AfterGenerateClass.wrap_dynamictable_columns(cls, sv) + cls = AfterGenerateClass.inject_elementidentifiers(cls, sv, self._get_element_import) + cls = AfterGenerateClass.strip_vector_data_slots(cls, sv) return cls def before_render_template(self, template: PydanticModule, sv: SchemaView) -> PydanticModule: @@ -204,15 +211,17 @@ def make_array_anyofs(slot: SlotResult) -> SlotResult: # merge injects/imports from the numpydantic array without using the merge method if slot.injected_classes is None: slot.injected_classes = NumpydanticArray.INJECTS.copy() - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.injected_classes.extend(NumpydanticArray.INJECTS.copy()) - if isinstance(slot.imports, list): + if isinstance( + slot.imports, list + ): # pragma: no cover - for completeness, shouldn't happen slot.imports = ( Imports(imports=slot.imports) + NumpydanticArray.IMPORTS.model_copy() ) elif isinstance(slot.imports, Imports): slot.imports += NumpydanticArray.IMPORTS.model_copy() - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.imports = NumpydanticArray.IMPORTS.model_copy() return slot @@ -224,17 +233,20 @@ def make_named_class_range(slot: SlotResult) -> SlotResult: """ if "named" in slot.source.annotations and slot.source.annotations["named"].value: - slot.attribute.range = f"Named[{slot.attribute.range}]" + + slot.attribute.range = wrap_preserving_optional(slot.attribute.range, "Named") named_injects = [ModelTypeString, _get_name, NamedString] if slot.injected_classes is None: slot.injected_classes = named_injects - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.injected_classes.extend([ModelTypeString, _get_name, NamedString]) - if isinstance(slot.imports, list): + if isinstance( + slot.imports, list + ): # pragma: no cover - for completeness, shouldn't happen slot.imports = Imports(imports=slot.imports) + NamedImports elif isinstance(slot.imports, Imports): slot.imports += NamedImports - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.imports = NamedImports return slot @@ -254,41 +266,57 @@ def inject_dynamictable(cls: ClassResult) -> ClassResult: Returns: """ - if cls.cls.name in "DynamicTable": - cls.cls.bases = ["DynamicTableMixin"] + if cls.cls.name == "DynamicTable": + cls.cls.bases = ["DynamicTableMixin", "ConfiguredBaseModel"] - if cls.injected_classes is None: + if ( + cls.injected_classes is None + ): # pragma: no cover - for completeness, shouldn't happen cls.injected_classes = DYNAMIC_TABLE_INJECTS.copy() else: cls.injected_classes.extend(DYNAMIC_TABLE_INJECTS.copy()) if isinstance(cls.imports, Imports): cls.imports += DYNAMIC_TABLE_IMPORTS - elif isinstance(cls.imports, list): + elif isinstance( + cls.imports, list + ): # pragma: no cover - for completeness, shouldn't happen cls.imports = Imports(imports=cls.imports) + DYNAMIC_TABLE_IMPORTS - else: + else: # pragma: no cover - for completeness, shouldn't happen cls.imports = DYNAMIC_TABLE_IMPORTS.model_copy() elif cls.cls.name == "VectorData": - cls.cls.bases = ["VectorDataMixin"] + cls.cls.bases = ["VectorDataMixin", "ConfiguredBaseModel"] + # make ``value`` generic on T + if "value" in cls.cls.attributes: + cls.cls.attributes["value"].range = "Optional[T]" elif cls.cls.name == "VectorIndex": - cls.cls.bases = ["VectorIndexMixin"] + cls.cls.bases = ["VectorIndexMixin", "ConfiguredBaseModel"] elif cls.cls.name == "DynamicTableRegion": - cls.cls.bases = ["DynamicTableRegionMixin", "VectorData"] + cls.cls.bases = ["DynamicTableRegionMixin", "VectorData", "ConfiguredBaseModel"] elif cls.cls.name == "AlignedDynamicTable": cls.cls.bases = ["AlignedDynamicTableMixin", "DynamicTable"] + elif cls.cls.name == "ElementIdentifiers": + cls.cls.bases = ["ElementIdentifiersMixin", "Data", "ConfiguredBaseModel"] + # make ``value`` generic on T + if "value" in cls.cls.attributes: + cls.cls.attributes["value"].range = "Optional[T]" elif cls.cls.name == "TimeSeriesReferenceVectorData": # in core.nwb.base, so need to inject and import again cls.cls.bases = ["TimeSeriesReferenceVectorDataMixin", "VectorData"] - if cls.injected_classes is None: + if ( + cls.injected_classes is None + ): # pragma: no cover - for completeness, shouldn't happen cls.injected_classes = TSRVD_INJECTS.copy() else: cls.injected_classes.extend(TSRVD_INJECTS.copy()) if isinstance(cls.imports, Imports): cls.imports += TSRVD_IMPORTS - elif isinstance(cls.imports, list): + elif isinstance( + cls.imports, list + ): # pragma: no cover - for completeness, shouldn't happen cls.imports = Imports(imports=cls.imports) + TSRVD_IMPORTS - else: + else: # pragma: no cover - for completeness, shouldn't happen cls.imports = TSRVD_IMPORTS.model_copy() return cls @@ -305,34 +333,60 @@ def wrap_dynamictable_columns(cls: ClassResult, sv: SchemaView) -> ClassResult: ): for an_attr in cls.cls.attributes: if "NDArray" in (slot_range := cls.cls.attributes[an_attr].range): - if an_attr.endswith("_index"): - cls.cls.attributes[an_attr].range = "".join( - ["VectorIndex[", slot_range, "]"] - ) - else: - cls.cls.attributes[an_attr].range = "".join( - ["VectorData[", slot_range, "]"] - ) + if an_attr == "id": + cls.cls.attributes[an_attr].range = "ElementIdentifiers" + return cls + + wrap_cls = "VectorIndex" if an_attr.endswith("_index") else "VectorData" + + cls.cls.attributes[an_attr].range = wrap_preserving_optional( + slot_range, wrap_cls + ) + + return cls + + @staticmethod + def inject_elementidentifiers( + cls: ClassResult, sv: SchemaView, import_method: Callable[[str], Import] + ) -> ClassResult: + """ + Inject ElementIdentifiers into module that define dynamictables - + needed to handle ID columns + """ + if ( + cls.source.is_a == "DynamicTable" + or "DynamicTable" in sv.class_ancestors(cls.source.name) + ) and sv.schema.name != "hdmf-common.table": + imp = import_method("ElementIdentifiers") + cls.imports += [imp] + return cls + + @staticmethod + def strip_vector_data_slots(cls: ClassResult, sv: SchemaView) -> ClassResult: + """ + Remove spurious ``vector_data`` slots from DynamicTables + """ + if "vector_data" in cls.cls.attributes: + del cls.cls.attributes["vector_data"] return cls -def compile_python( - text_or_fn: str, package_path: Path = None, module_name: str = "test" -) -> ModuleType: +def wrap_preserving_optional(annotation: str, wrap: str) -> str: """ - Compile the text or file and return the resulting module - @param text_or_fn: Python text or file name that references python file - @param package_path: Root package path. If omitted and we've got a python file, - the package is the containing - directory - @return: Compiled module + Add a wrapping type to a type annotation string, + preserving any `Optional[]` annotation, bumping it to the outside + + Examples: + + >>> wrap_preserving_optional('Optional[list[str]]', 'NewType') + 'Optional[NewType[list[str]]]' + """ - python_txt = file_text(text_or_fn) - if package_path is None and python_txt != text_or_fn: - package_path = Path(text_or_fn) - spec = compile(python_txt, "", "exec") - module = ModuleType(module_name) - - exec(spec, module.__dict__) - sys.modules[module_name] = module - return module + + is_optional = OPTIONAL_PATTERN.match(annotation) + if is_optional: + annotation = is_optional.groups()[0] + annotation = f"Optional[{wrap}[{annotation}]]" + else: + annotation = f"{wrap}[{annotation}]" + return annotation diff --git a/nwb_linkml/src/nwb_linkml/includes/base.py b/nwb_linkml/src/nwb_linkml/includes/base.py index ed69bf31..3ecae8c9 100644 --- a/nwb_linkml/src/nwb_linkml/includes/base.py +++ b/nwb_linkml/src/nwb_linkml/includes/base.py @@ -12,3 +12,38 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") """ + +BASEMODEL_COERCE_VALUE = """ + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + \"\"\"Try to rescue instantiation by using the value field\"\"\" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 +""" + +BASEMODEL_COERCE_CHILD = """ + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + \"\"\"Recast parent classes into child classes\"\"\" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v +""" diff --git a/nwb_linkml/src/nwb_linkml/includes/hdmf.py b/nwb_linkml/src/nwb_linkml/includes/hdmf.py index 8bc91073..7a7d2948 100644 --- a/nwb_linkml/src/nwb_linkml/includes/hdmf.py +++ b/nwb_linkml/src/nwb_linkml/includes/hdmf.py @@ -53,8 +53,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -138,7 +141,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -246,11 +249,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -259,6 +265,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -277,17 +284,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -320,9 +335,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -370,7 +385,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -571,10 +586,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -604,28 +622,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -633,8 +652,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -692,14 +710,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -733,7 +756,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self @@ -828,6 +851,13 @@ def __setitem__(self, key: Union[int, slice, Iterable], value: Any) -> None: ) +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + DYNAMIC_TABLE_IMPORTS = Imports( imports=[ Import(module="pandas", alias="pd"), @@ -871,6 +901,7 @@ def __setitem__(self, key: Union[int, slice, Iterable], value: Any) -> None: DynamicTableRegionMixin, DynamicTableMixin, AlignedDynamicTableMixin, + ElementIdentifiersMixin, ] TSRVD_IMPORTS = Imports( @@ -912,3 +943,8 @@ class TimeSeriesReferenceVectorData(TimeSeriesReferenceVectorDataMixin): """TimeSeriesReferenceVectorData subclass for testing""" pass + + class ElementIdentifiers(ElementIdentifiersMixin): + """ElementIdentifiers subclass for testing""" + + pass diff --git a/nwb_linkml/src/nwb_linkml/io/hdf5.py b/nwb_linkml/src/nwb_linkml/io/hdf5.py index ade89d95..bf4fbe6f 100644 --- a/nwb_linkml/src/nwb_linkml/io/hdf5.py +++ b/nwb_linkml/src/nwb_linkml/io/hdf5.py @@ -22,6 +22,7 @@ import json import os +import re import shutil import subprocess import sys @@ -31,11 +32,18 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Union, overload import h5py +import networkx as nx import numpy as np +from numpydantic.interface.hdf5 import H5ArrayPath from pydantic import BaseModel from tqdm import tqdm -from nwb_linkml.maps.hdf5 import ReadPhases, ReadQueue, flatten_hdf +from nwb_linkml.maps.hdf5 import ( + get_attr_references, + get_dataset_references, + get_references, + resolve_hardlink, +) if TYPE_CHECKING: from nwb_linkml.providers.schema import SchemaProvider @@ -47,6 +55,221 @@ from typing_extensions import Never +SKIP_PATTERN = re.compile("(^/specifications.*)|(\.specloc)") +"""Nodes to always skip in reading e.g. because they are handled elsewhere""" + + +def hdf_dependency_graph(h5f: Path | h5py.File | h5py.Group) -> nx.DiGraph: + """ + Directed dependency graph of dataset and group nodes in an NWBFile such that + each node ``n_i`` is connected to node ``n_j`` if + + * ``n_j`` is ``n_i``'s child + * ``n_i`` contains a reference to ``n_j`` + + Resolve references in + + * Attributes + * Dataset columns + * Compound dtypes + + Edges are labeled with ``reference`` or ``child`` depending on the type of edge it is, + and attributes from the hdf5 file are added as node attributes. + + Args: + h5f (:class:`pathlib.Path` | :class:`h5py.File`): NWB file to graph + + Returns: + :class:`networkx.DiGraph` + """ + + if isinstance(h5f, (Path, str)): + h5f = h5py.File(h5f, "r") + + g = nx.DiGraph() + + def _visit_item(name: str, node: h5py.Dataset | h5py.Group) -> None: + if SKIP_PATTERN.match(node.name): + return + # find references in attributes + refs = get_references(node) + # add edges from references + edges = [(node.name, ref) for ref in refs if not SKIP_PATTERN.match(ref)] + g.add_edges_from(edges, label="reference") + + # add children, if group + if isinstance(node, h5py.Group): + children = [ + resolve_hardlink(child) + for child in node.values() + if not SKIP_PATTERN.match(child.name) + ] + edges = [(node.name, ref) for ref in children if not SKIP_PATTERN.match(ref)] + g.add_edges_from(edges, label="child") + + # ensure node added to graph + if len(edges) == 0: + g.add_node(node.name) + + # store attrs in node + g.nodes[node.name].update(node.attrs) + + # apply to root + _visit_item(h5f.name, h5f) + + h5f.visititems(_visit_item) + return g + + +def filter_dependency_graph(g: nx.DiGraph) -> nx.DiGraph: + """ + Remove nodes from a dependency graph if they + + * have no neurodata type AND + * have no outbound edges + + OR + + * are a VectorIndex (which are handled by the dynamictable mixins) + """ + remove_nodes = [] + node: str + for node in g.nodes: + ndtype = g.nodes[node].get("neurodata_type", None) + if (ndtype is None and g.out_degree(node) == 0) or SKIP_PATTERN.match(node): + remove_nodes.append(node) + + g.remove_nodes_from(remove_nodes) + return g + + +def _load_node( + path: str, h5f: h5py.File, provider: "SchemaProvider", context: dict +) -> dict | BaseModel: + """ + Load an individual node in the graph, then removes it from the graph + Args: + path: + g: + context: + + Returns: + + """ + obj = h5f.get(path) + + if isinstance(obj, h5py.Dataset): + args = _load_dataset(obj, h5f, context) + elif isinstance(obj, h5py.Group): + args = _load_group(obj, h5f, context) + else: + raise TypeError(f"Nodes can only be h5py Datasets and Groups, got {obj}") + + if "neurodata_type" in obj.attrs: + model = provider.get_class(obj.attrs["namespace"], obj.attrs["neurodata_type"]) + return model(**args) + else: + if "name" in args: + del args["name"] + if "hdf5_path" in args: + del args["hdf5_path"] + return args + + +def _load_dataset( + dataset: h5py.Dataset, h5f: h5py.File, context: dict +) -> Union[dict, str, int, float]: + """ + Resolves datasets that do not have a ``neurodata_type`` as a dictionary or a scalar. + + If the dataset is a single value without attrs, load it and return as a scalar value. + Otherwise return a :class:`.H5ArrayPath` as a reference to the dataset in the `value` key. + """ + res = {} + if dataset.shape == (): + val = dataset[()] + if isinstance(val, h5py.h5r.Reference): + val = context.get(h5f[val].name) + # if this is just a scalar value, return it + if not dataset.attrs: + return val + + res["value"] = val + elif len(dataset) > 0 and isinstance(dataset[0], h5py.h5r.Reference): + # vector of references + res["value"] = [context.get(h5f[ref].name) for ref in dataset[:]] + elif len(dataset.dtype) > 1: + # compound dataset - check if any of the fields are references + for name in dataset.dtype.names: + if isinstance(dataset[name][0], h5py.h5r.Reference): + res[name] = [context.get(h5f[ref].name) for ref in dataset[name]] + else: + res[name] = H5ArrayPath(h5f.filename, dataset.name, name) + else: + res["value"] = H5ArrayPath(h5f.filename, dataset.name) + + res.update(dataset.attrs) + if "namespace" in res: + del res["namespace"] + if "neurodata_type" in res: + del res["neurodata_type"] + res["name"] = dataset.name.split("/")[-1] + res["hdf5_path"] = dataset.name + + # resolve attr references + for k, v in res.items(): + if isinstance(v, h5py.h5r.Reference): + ref_path = h5f[v].name + if SKIP_PATTERN.match(ref_path): + res[k] = ref_path + else: + res[k] = context[ref_path] + + if len(res) == 1: + return res["value"] + else: + return res + + +def _load_group(group: h5py.Group, h5f: h5py.File, context: dict) -> dict: + """ + Load a group! + """ + res = {} + res.update(group.attrs) + for child_name, child in group.items(): + if child.name in context: + res[child_name] = context[child.name] + elif isinstance(child, h5py.Dataset): + res[child_name] = _load_dataset(child, h5f, context) + elif isinstance(child, h5py.Group): + res[child_name] = _load_group(child, h5f, context) + else: + raise TypeError( + "Can only handle preinstantiated child objects in context, datasets, and group," + f" got {child} for {child_name}" + ) + if "namespace" in res: + del res["namespace"] + if "neurodata_type" in res: + del res["neurodata_type"] + name = group.name.split("/")[-1] + if name: + res["name"] = name + res["hdf5_path"] = group.name + + # resolve attr references + for k, v in res.items(): + if isinstance(v, h5py.h5r.Reference): + ref_path = h5f[v].name + if SKIP_PATTERN.match(ref_path): + res[k] = ref_path + else: + res[k] = context[ref_path] + + return res + + class HDF5IO: """ Read (and eventually write) from an NWB HDF5 file. @@ -106,28 +329,22 @@ def read(self, path: Optional[str] = None) -> Union["NWBFile", BaseModel, Dict[s h5f = h5py.File(str(self.path)) src = h5f.get(path) if path else h5f - - # get all children of selected item - if isinstance(src, (h5py.File, h5py.Group)): - children = flatten_hdf(src) - else: - raise NotImplementedError("directly read individual datasets") - - queue = ReadQueue(h5f=self.path, queue=children, provider=provider) - - # Apply initial planning phase of reading - queue.apply_phase(ReadPhases.plan) - # Read operations gather the data before casting into models - queue.apply_phase(ReadPhases.read) - # Construction operations actually cast the models - # this often needs to run several times as models with dependencies wait for their - # dependents to be cast - queue.apply_phase(ReadPhases.construct) + graph = hdf_dependency_graph(src) + graph = filter_dependency_graph(graph) + + # topo sort to get read order + # TODO: This could be parallelized using `topological_generations`, + # but it's not clear what the perf bonus would be because there are many generations + # with few items + topo_order = list(reversed(list(nx.topological_sort(graph)))) + context = {} + for node in topo_order: + res = _load_node(node, h5f, provider, context) + context[node] = res if path is None: - return queue.completed["/"].result - else: - return queue.completed[path].result + path = "/" + return context[path] def write(self, path: Path) -> Never: """ @@ -167,7 +384,7 @@ def make_provider(self) -> "SchemaProvider": """ from nwb_linkml.providers.schema import SchemaProvider - h5f = h5py.File(str(self.path)) + h5f = h5py.File(str(self.path), "r") schema = read_specs_as_dicts(h5f.get("specifications")) # get versions for each namespace @@ -269,7 +486,7 @@ def _find_references(name: str, obj: h5py.Group | h5py.Dataset) -> None: return references -def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> Path: +def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> Path | None: """ Create a truncated HDF5 file where only the first few samples are kept. @@ -285,6 +502,14 @@ def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> P Returns: :class:`pathlib.Path` path of the truncated file """ + if shutil.which("h5repack") is None: + warnings.warn( + "Truncation requires h5repack to be available, " + "or else the truncated files will be no smaller than the originals", + stacklevel=2, + ) + return + target = source.parent / (source.stem + "_truncated.hdf5") if target is None else Path(target) source = Path(source) @@ -300,17 +525,34 @@ def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> P os.chmod(target, 0o774) to_resize = [] + attr_refs = {} + dataset_refs = {} def _need_resizing(name: str, obj: h5py.Dataset | h5py.Group) -> None: if isinstance(obj, h5py.Dataset) and obj.size > n: to_resize.append(name) - print("Resizing datasets...") + def _find_attr_refs(name: str, obj: h5py.Dataset | h5py.Group) -> None: + """Find all references in object attrs""" + refs = get_attr_references(obj) + if refs: + attr_refs[name] = refs + + def _find_dataset_refs(name: str, obj: h5py.Dataset | h5py.Group) -> None: + """Find all references in datasets themselves""" + refs = get_dataset_references(obj) + if refs: + dataset_refs[name] = refs + # first we get the items that need to be resized and then resize them below # problems with writing to the file from within the visititems call + print("Planning resize...") h5f_target = h5py.File(str(target), "r+") h5f_target.visititems(_need_resizing) + h5f_target.visititems(_find_attr_refs) + h5f_target.visititems(_find_dataset_refs) + print("Resizing datasets...") for resize in to_resize: obj = h5f_target.get(resize) try: @@ -320,10 +562,14 @@ def _need_resizing(name: str, obj: h5py.Dataset | h5py.Group) -> None: # so we have to copy and create a new dataset tmp_name = obj.name + "__tmp" original_name = obj.name + obj.parent.move(obj.name, tmp_name) old_obj = obj.parent.get(tmp_name) - new_obj = obj.parent.create_dataset(original_name, data=old_obj[0:n]) + new_obj = obj.parent.create_dataset( + original_name, data=old_obj[0:n], dtype=old_obj.dtype + ) for k, v in old_obj.attrs.items(): + new_obj.attrs[k] = v del new_obj.parent[tmp_name] @@ -331,16 +577,18 @@ def _need_resizing(name: str, obj: h5py.Dataset | h5py.Group) -> None: h5f_target.close() # use h5repack to actually remove the items from the dataset - if shutil.which("h5repack") is None: - warnings.warn( - "Truncated file made, but since h5repack not found in path, file won't be any smaller", - stacklevel=2, - ) - return target - print("Repacking hdf5...") res = subprocess.run( - ["h5repack", "-f", "GZIP=9", str(target), str(target_tmp)], capture_output=True + [ + "h5repack", + "--verbose=2", + "--enable-error-stack", + "-f", + "GZIP=9", + str(target), + str(target_tmp), + ], + capture_output=True, ) if res.returncode != 0: warnings.warn(f"h5repack did not return 0: {res.stderr} {res.stdout}", stacklevel=2) @@ -348,6 +596,36 @@ def _need_resizing(name: str, obj: h5py.Dataset | h5py.Group) -> None: target_tmp.unlink() return target + h5f_target = h5py.File(str(target_tmp), "r+") + + # recreate references after repacking, because repacking ruins them if they + # are in a compound dtype + for obj_name, obj_refs in attr_refs.items(): + obj = h5f_target.get(obj_name) + for attr_name, ref_target in obj_refs.items(): + ref_target = h5f_target.get(ref_target) + obj.attrs[attr_name] = ref_target.ref + + for obj_name, obj_refs in dataset_refs.items(): + obj = h5f_target.get(obj_name) + if isinstance(obj_refs, list): + if len(obj_refs) == 1: + ref_target = h5f_target.get(obj_refs[0]) + obj[()] = ref_target.ref + else: + targets = [h5f_target.get(ref).ref for ref in obj_refs[:n]] + obj[:] = targets + else: + # dict for a compound dataset + for col_name, column_refs in obj_refs.items(): + targets = [h5f_target.get(ref).ref for ref in column_refs[:n]] + data = obj[:] + data[col_name] = targets + obj[:] = data + + h5f_target.flush() + h5f_target.close() + target.unlink() target_tmp.rename(target) diff --git a/nwb_linkml/src/nwb_linkml/io/schema.py b/nwb_linkml/src/nwb_linkml/io/schema.py index 42718f50..8f960c75 100644 --- a/nwb_linkml/src/nwb_linkml/io/schema.py +++ b/nwb_linkml/src/nwb_linkml/io/schema.py @@ -131,6 +131,8 @@ def load_namespace_adapter( else: adapter = NamespacesAdapter(namespaces=namespaces, schemas=sch) + adapter.populate_imports() + return adapter diff --git a/nwb_linkml/src/nwb_linkml/maps/dtype.py b/nwb_linkml/src/nwb_linkml/maps/dtype.py index d618dbe4..2497a659 100644 --- a/nwb_linkml/src/nwb_linkml/maps/dtype.py +++ b/nwb_linkml/src/nwb_linkml/maps/dtype.py @@ -3,7 +3,7 @@ """ from datetime import datetime -from typing import Any +from typing import Any, Optional import numpy as np @@ -160,14 +160,28 @@ def handle_dtype(dtype: DTypeType | None) -> str: elif isinstance(dtype, FlatDtype): return dtype.value elif isinstance(dtype, list) and isinstance(dtype[0], CompoundDtype): - # there is precisely one class that uses compound dtypes: - # TimeSeriesReferenceVectorData - # compoundDtypes are able to define a ragged table according to the schema - # but are used in this single case equivalently to attributes. - # so we'll... uh... treat them as slots. - # TODO + # Compound Dtypes are handled by the MapCompoundDtype dataset map, + # but this function is also used within ``check`` methods, so we should always + # return something from it rather than raise return "AnyType" else: # flat dtype return dtype + + +def inlined(dtype: DTypeType | None) -> Optional[bool]: + """ + Check if a slot should be inlined based on its dtype + + for now that is equivalent to checking whether that dtype is another a reference dtype, + but the function remains semantically reserved for answering this question w.r.t. dtype. + + Returns ``None`` if not inlined to not clutter generated models with unnecessary props + """ + return ( + True + if isinstance(dtype, ReferenceDtype) + or (isinstance(dtype, CompoundDtype) and isinstance(dtype.dtype, ReferenceDtype)) + else None + ) diff --git a/nwb_linkml/src/nwb_linkml/maps/hdf5.py b/nwb_linkml/src/nwb_linkml/maps/hdf5.py index a7b052fc..a507678c 100644 --- a/nwb_linkml/src/nwb_linkml/maps/hdf5.py +++ b/nwb_linkml/src/nwb_linkml/maps/hdf5.py @@ -5,832 +5,47 @@ so we will make our own mapping class here and re-evaluate whether they should be unified later """ -# FIXME: return and document whatever is left of this godforsaken module after refactoring # ruff: noqa: D102 # ruff: noqa: D101 -import contextlib -import datetime -import inspect -import sys -from abc import abstractmethod -from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import List, Union import h5py -from numpydantic.interface.hdf5 import H5ArrayPath -from pydantic import BaseModel, ConfigDict, Field -from nwb_linkml.annotations import unwrap_optional -from nwb_linkml.maps import Map -from nwb_linkml.types.hdf5 import HDF5_Path -if sys.version_info.minor >= 11: - from enum import StrEnum -else: - from enum import Enum - - class StrEnum(str, Enum): - """StrEnum-ish class for python 3.10""" - - -if TYPE_CHECKING: - from nwb_linkml.providers.schema import SchemaProvider - - -class ReadPhases(StrEnum): - plan = "plan" - """Before reading starts, building an index of objects to read""" - read = "read" - """Main reading operation""" - construct = "construct" - """After reading, casting the results of the read into their models""" - - -class H5SourceItem(BaseModel): - """ - Descriptor of items for each element when :func:`.flatten_hdf` flattens an hdf5 file. - - Consumed by :class:`.HDF5Map` classes, orchestrated by :class:`.ReadQueue` - """ - - path: str - """Absolute hdf5 path of element""" - h5f_path: str - """Path to the source hdf5 file""" - leaf: bool - """ - If ``True``, this item has no children - (and thus we should start instantiating it before ascending to parent classes) - """ - h5_type: Literal["group", "dataset"] - """What kind of hdf5 element this is""" - depends: List[str] = Field(default_factory=list) - """ - Paths of other source items that this item depends on before it can be instantiated. - eg. from softlinks - """ - attrs: dict = Field(default_factory=dict) - """Any static attrs that can be had from the element""" - namespace: Optional[str] = None - """Optional: The namespace that the neurodata type belongs to""" - neurodata_type: Optional[str] = None - """Optional: the neurodata type for this dataset or group""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - @property - def parts(self) -> List[str]: - """path split by /""" - return self.path.split("/") - - -class H5ReadResult(BaseModel): - """ - Result returned by each of our mapping operations. - - Also used as the source for operations in the ``construct`` :class:`.ReadPhases` - """ - - path: str - """absolute hdf5 path of element""" - source: Union[H5SourceItem, "H5ReadResult"] - """ - Source that this result is based on. - The map can modify this item, so the container should update the source - queue on each pass +def get_attr_references(obj: h5py.Dataset | h5py.Group) -> dict[str, str]: """ - completed: bool = False + Get any references in object attributes """ - Was this item completed by this map step? False for cases where eg. - we still have dependencies that need to be completed before this one - """ - result: Optional[dict | str | int | float | BaseModel] = None - """ - If completed, built result. A dict that can be instantiated into the model. - If completed is True and result is None, then remove this object - """ - model: Optional[Type[BaseModel]] = None - """ - The model that this item should be cast into - """ - completes: List[HDF5_Path] = Field(default_factory=list) - """ - If this result completes any other fields, we remove them from the build queue. - """ - namespace: Optional[str] = None - """ - Optional: the namespace of the neurodata type for this object - """ - neurodata_type: Optional[str] = None - """ - Optional: The neurodata type to use for this object - """ - applied: List[str] = Field(default_factory=list) - """ - Which map operations were applied to this item - """ - errors: List[str] = Field(default_factory=list) - """ - Problems that occurred during resolution - """ - depends: List[HDF5_Path] = Field(default_factory=list) - """ - Other items that the final resolution of this item depends on - """ - + refs = { + k: obj.file.get(ref).name + for k, ref in obj.attrs.items() + if isinstance(ref, h5py.h5r.Reference) + } + return refs -FlatH5 = Dict[str, H5SourceItem] - -class HDF5Map(Map): - phase: ReadPhases - priority: int = 0 +def get_dataset_references(obj: h5py.Dataset | h5py.Group) -> list[str] | dict[str, str]: """ - Within a phase, sort mapping operations from low to high priority - (maybe this should be renamed because highest priority last doesn't make a lot of sense) - """ - - @classmethod - @abstractmethod - def check( - cls, - src: H5SourceItem | H5ReadResult, - provider: "SchemaProvider", - completed: Dict[str, H5ReadResult], - ) -> bool: - """Check if this map applies to the given item to read""" - - @classmethod - @abstractmethod - def apply( - cls, - src: H5SourceItem | H5ReadResult, - provider: "SchemaProvider", - completed: Dict[str, H5ReadResult], - ) -> H5ReadResult: - """Actually apply the map!""" - - -# -------------------------------------------------- -# Planning maps -# -------------------------------------------------- - - -def check_empty(obj: h5py.Group) -> bool: - """ - Check if a group has no attrs or children OR has no attrs and all its children - also have no attrs and no children - - Returns: - bool + Get references in datasets """ + refs = [] + # For datasets, apply checks depending on shape of data. if isinstance(obj, h5py.Dataset): - return False - - # check if we are empty - no_attrs = False - if len(obj.attrs) == 0: - no_attrs = True - - no_children = False - if len(obj.keys()) == 0: - no_children = True - - # check if immediate children are empty - # handles empty groups of empty groups - children_empty = False - if all( - [ - isinstance(item, h5py.Group) and len(item.keys()) == 0 and len(item.attrs) == 0 - for item in obj.values() - ] - ): - children_empty = True - - # if we have no attrs and we are a leaf OR our children are empty, remove us - return bool(no_attrs and (no_children or children_empty)) - - -class PruneEmpty(HDF5Map): - """Remove groups with no attrs""" - - phase = ReadPhases.plan - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "group": - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return check_empty(obj) - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - return H5ReadResult.model_construct(path=src.path, source=src, completed=True) - - -# -# class ResolveDynamicTable(HDF5Map): -# """ -# Handle loading a dynamic table! -# -# Dynamic tables are sort of odd in that their models don't include their fields -# (except as a list of strings in ``colnames`` ), -# so we need to create a new model that includes fields for each column, -# and then we include the datasets as :class:`~numpydantic.interface.hdf5.H5ArrayPath` -# objects which lazy load the arrays in a thread/process safe way. -# -# This map also resolves the child elements, -# indicating so by the ``completes`` field in the :class:`.ReadResult` -# """ -# -# phase = ReadPhases.read -# priority = 1 -# -# @classmethod -# def check( -# cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] -# ) -> bool: -# if src.h5_type == "dataset": -# return False -# if "neurodata_type" in src.attrs: -# if src.attrs["neurodata_type"] == "DynamicTable": -# return True -# # otherwise, see if it's a subclass -# model = provider.get_class(src.attrs["namespace"], src.attrs["neurodata_type"]) -# # just inspect the MRO as strings rather than trying to check subclasses because -# # we might replace DynamicTable in the future, and there isn't a stable DynamicTable -# # class to inherit from anyway because of the whole multiple versions thing -# parents = [parent.__name__ for parent in model.__mro__] -# return "DynamicTable" in parents -# else: -# return False -# -# @classmethod -# def apply( -# cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] -# ) -> H5ReadResult: -# with h5py.File(src.h5f_path, "r") as h5f: -# obj = h5f.get(src.path) -# -# # make a populated model :) -# base_model = provider.get_class(src.namespace, src.neurodata_type) -# model = dynamictable_to_model(obj, base=base_model) -# -# completes = [HDF5_Path(child.name) for child in obj.values()] -# -# return H5ReadResult( -# path=src.path, -# source=src, -# result=model, -# completes=completes, -# completed=True, -# applied=["ResolveDynamicTable"], -# ) - - -class ResolveModelGroup(HDF5Map): - """ - HDF5 Groups that have a model, as indicated by ``neurodata_type`` in their attrs. - We use the model to determine what fields we should get, and then stash references - to the children to process later as :class:`.HDF5_Path` - - **Special Case:** Some groups like ``ProcessingGroup`` and others that have an arbitrary - number of named children have a special ``children`` field that is a dictionary mapping - names to the objects themselves. - - So for example, this: - - /processing/ - eye_tracking/ - cr_ellipse_fits/ - center_x - center_y - ... - eye_ellipse_fits/ - ... - pupil_ellipse_fits/ - ... - eye_tracking_rig_metadata/ - ... - - would pack the ``eye_tracking`` group (a ``ProcessingModule`` ) as: - - { - "name": "eye_tracking", - "children": { - "cr_ellipse_fits": HDF5_Path('/processing/eye_tracking/cr_ellipse_fits'), - "eye_ellipse_fits" : HDF5_Path('/processing/eye_tracking/eye_ellipse_fits'), - ... - } - } - - We will do some nice things in the model metaclass to make it possible to access the children - like ``nwbfile.processing.cr_ellipse_fits.center_x`` - rather than having to switch between indexing and attribute access :) - """ - - phase = ReadPhases.read - priority = 10 # do this generally last - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return bool("neurodata_type" in src.attrs and src.h5_type == "group") - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - model = provider.get_class(src.namespace, src.neurodata_type) - res = {} - depends = [] - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - for key in model.model_fields: - if key == "children": - res[key] = {name: resolve_hardlink(child) for name, child in obj.items()} - depends.extend([resolve_hardlink(child) for child in obj.values()]) - elif key in obj.attrs: - res[key] = obj.attrs[key] - continue - elif key in obj: - # make sure it's not empty - if check_empty(obj[key]): - continue - # stash a reference to this, we'll compile it at the end - depends.append(resolve_hardlink(obj[key])) - res[key] = resolve_hardlink(obj[key]) - - res["hdf5_path"] = src.path - res["name"] = src.parts[-1] - return H5ReadResult( - path=src.path, - source=src, - completed=True, - result=res, - model=model, - namespace=src.namespace, - neurodata_type=src.neurodata_type, - applied=["ResolveModelGroup"], - depends=depends, - ) - - -class ResolveDatasetAsDict(HDF5Map): - """ - Resolve datasets that do not have a ``neurodata_type`` of their own as a dictionary - that will be packaged into a model in the next step. Grabs the array in an - :class:`~numpydantic.interface.hdf5.H5ArrayPath` - under an ``array`` key, and then grabs any additional ``attrs`` as well. - - Mutually exclusive with :class:`.ResolveScalars` - this only applies to datasets that are larger - than a single entry. - """ - - phase = ReadPhases.read - priority = 11 - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "dataset" and "neurodata_type" not in src.attrs: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return obj.shape != () - else: - return False - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - - res = { - "array": H5ArrayPath(file=src.h5f_path, path=src.path), - "hdf5_path": src.path, - "name": src.parts[-1], - **src.attrs, - } - return H5ReadResult( - path=src.path, source=src, completed=True, result=res, applied=["ResolveDatasetAsDict"] - ) - - -class ResolveScalars(HDF5Map): - phase = ReadPhases.read - priority = 11 # catchall - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "dataset" and "neurodata_type" not in src.attrs: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return obj.shape == () - else: - return False - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - res = obj[()] - return H5ReadResult( - path=src.path, source=src, completed=True, result=res, applied=["ResolveScalars"] - ) - - -class ResolveContainerGroups(HDF5Map): - """ - Groups like ``/acquisition``` and others that have no ``neurodata_type`` - (and thus no model) are returned as a dictionary with :class:`.HDF5_Path` references to - the children they contain - """ - - phase = ReadPhases.read - priority = 9 - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "group" and "neurodata_type" not in src.attrs and len(src.attrs) == 0: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return len(obj.keys()) > 0 - else: - return False - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - """Simple, just return a dict with references to its children""" - depends = [] - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - children = {} - for k, v in obj.items(): - children[k] = HDF5_Path(v.name) - depends.append(HDF5_Path(v.name)) - - # res = { - # 'name': src.parts[-1], - # 'hdf5_path': src.path, - # **children - # } - - return H5ReadResult( - path=src.path, - source=src, - completed=True, - result=children, - depends=depends, - applied=["ResolveContainerGroups"], - ) - - -# -------------------------------------------------- -# Completion Steps -# -------------------------------------------------- - - -class CompletePassThrough(HDF5Map): - """ - Passthrough map for the construction phase for models that don't need any more work done - - - :class:`.ResolveDynamicTable` - - :class:`.ResolveDatasetAsDict` - - :class:`.ResolveScalars` - """ - - phase = ReadPhases.construct - priority = 1 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - passthrough_ops = ("ResolveDynamicTable", "ResolveDatasetAsDict", "ResolveScalars") - - return any(hasattr(src, "applied") and op in src.applied for op in passthrough_ops) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - return src - - -class CompleteContainerGroups(HDF5Map): - """ - Complete container groups (usually top-level groups like /acquisition) - that do not have a ndueodata type of their own by resolving them as dictionaries - of values (that will then be given to their parent model) - - """ - - phase = ReadPhases.construct - priority = 3 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return ( - src.model is None - and src.neurodata_type is None - and src.source.h5_type == "group" - and all([depend in completed for depend in src.depends]) - ) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - res, errors, completes = resolve_references(src.result, completed) - - return H5ReadResult( - result=res, - errors=errors, - completes=completes, - **src.model_dump(exclude={"result", "errors", "completes"}), - ) - - -class CompleteModelGroups(HDF5Map): - phase = ReadPhases.construct - priority = 4 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return ( - src.model is not None - and src.source.h5_type == "group" - and src.neurodata_type != "NWBFile" - and all([depend in completed for depend in src.depends]) - ) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - # gather any results that were left for completion elsewhere - # first get all already-completed items - res = {k: v for k, v in src.result.items() if not isinstance(v, HDF5_Path)} - unpacked_results, errors, completes = resolve_references(src.result, completed) - res.update(unpacked_results) - - # now that we have the model in hand, we can solve any datasets that had an array - # but whose attributes are fixed (and thus should just be an array, rather than a subclass) - for k, v in src.model.model_fields.items(): - annotation = unwrap_optional(v.annotation) - if ( - inspect.isclass(annotation) - and not issubclass(annotation, BaseModel) - and isinstance(res, dict) - and k in res - and isinstance(res[k], dict) - and "array" in res[k] - ): - res[k] = res[k]["array"] - - instance = src.model(**res) - return H5ReadResult( - path=src.path, - source=src, - result=instance, - model=src.model, - completed=True, - completes=completes, - neurodata_type=src.neurodata_type, - namespace=src.namespace, - applied=src.applied + ["CompleteModelGroups"], - errors=errors, - ) - - -class CompleteNWBFile(HDF5Map): - """ - The Top-Level NWBFile class is so special cased we just make its own completion special case! - - .. todo:: - - This is truly hideous, just meant as a way to get to the finish line on a late night, - will be cleaned up later - - """ - - phase = ReadPhases.construct - priority = 11 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return src.neurodata_type == "NWBFile" and all( - [depend in completed for depend in src.depends] - ) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - res = {k: v for k, v in src.result.items() if not isinstance(v, HDF5_Path)} - unpacked_results, errors, completes = resolve_references(src.result, completed) - res.update(unpacked_results) - - res["name"] = "root" - res["file_create_date"] = [ - datetime.datetime.fromisoformat(ts.decode("utf-8")) - for ts in res["file_create_date"]["array"][:] - ] - if "stimulus" not in res: - res["stimulus"] = provider.get_class("core", "NWBFileStimulus")() - electrode_groups = [] - egroup_keys = list(res["general"].get("extracellular_ephys", {}).keys()) - egroup_dict = {} - for k in egroup_keys: - if k != "electrodes": - egroup = res["general"]["extracellular_ephys"][k] - electrode_groups.append(egroup) - egroup_dict[egroup.hdf5_path] = egroup - del res["general"]["extracellular_ephys"][k] - if len(electrode_groups) > 0: - res["general"]["extracellular_ephys"]["electrode_group"] = electrode_groups - trode_type = provider.get_class("core", "NWBFileGeneralExtracellularEphysElectrodes") - # anmro = list(type(res['general']['extracellular_ephys']['electrodes']).__mro__) - # anmro.insert(1, trode_type) - trodes_original = res["general"]["extracellular_ephys"]["electrodes"] - trodes = trode_type.model_construct(trodes_original.model_dump()) - res["general"]["extracellular_ephys"]["electrodes"] = trodes - - instance = src.model(**res) - return H5ReadResult( - path=src.path, - source=src, - result=instance, - model=src.model, - completed=True, - completes=completes, - neurodata_type=src.neurodata_type, - namespace=src.namespace, - applied=src.applied + ["CompleteModelGroups"], - errors=errors, - ) - - -class ReadQueue(BaseModel): - """Container model to store items as they are built""" - - h5f: Path = Field( - description=( - "Path to the source hdf5 file used when resolving the queue! " - "Each translation step should handle opening and closing the file, " - "rather than passing a handle around" - ) - ) - provider: "SchemaProvider" = Field( - description="SchemaProvider used by each of the items in the read queue" - ) - queue: Dict[str, H5SourceItem | H5ReadResult] = Field( - default_factory=dict, - description="Items left to be instantiated, keyed by hdf5 path", - ) - completed: Dict[str, H5ReadResult] = Field( - default_factory=dict, - description="Items that have already been instantiated, keyed by hdf5 path", - ) - model_config = ConfigDict(arbitrary_types_allowed=True) - phases_completed: List[ReadPhases] = Field( - default_factory=list, description="Phases that have already been completed" - ) - - def apply_phase(self, phase: ReadPhases, max_passes: int = 5) -> None: - phase_maps = [m for m in HDF5Map.__subclasses__() if m.phase == phase] - phase_maps = sorted(phase_maps, key=lambda x: x.priority) - - results = [] - - # TODO: Thread/multiprocess this - for item in self.queue.values(): - for op in phase_maps: - if op.check(item, self.provider, self.completed): - # Formerly there was an "exclusive" property in the maps which let - # potentially multiple operations be applied per stage, - # except if an operation was `exclusive` which would break - # iteration over the operations. - # This was removed because it was badly implemented, - # but if there is ever a need to do that, - # then we would need to decide what to do with the multiple results. - results.append(op.apply(item, self.provider, self.completed)) - break # out of inner iteration - - # remake the source queue and save results - completes = [] - for res in results: - # remove the original item - del self.queue[res.path] - if res.completed: - # if the item has been finished and there is some result, add it to the results - if res.result is not None: - self.completed[res.path] = res - # otherwise if the item has been completed and there was no result, - # just drop it. - - # if we have completed other things, delete them from the queue - completes.extend(res.completes) - - else: - # if we didn't complete the item (eg. we found we needed more dependencies), - # add the updated source to the queue again - if phase != ReadPhases.construct: - self.queue[res.path] = res.source - else: - self.queue[res.path] = res - - # delete the ones that were already completed but might have been - # incorrectly added back in the pile - for c in completes: - with contextlib.suppress(KeyError): - del self.queue[c] - - # if we have nothing left in our queue, we have completed this phase - # and prepare only ever has one pass - if phase == ReadPhases.plan: - self.phases_completed.append(phase) - return - - if len(self.queue) == 0: - self.phases_completed.append(phase) - if phase != ReadPhases.construct: - # if we're not in the last phase, move our completed to our queue - self.queue = self.completed - self.completed = {} - elif max_passes > 0: - self.apply_phase(phase, max_passes=max_passes - 1) - - -def flatten_hdf( - h5f: h5py.File | h5py.Group, skip: str = "specifications" -) -> Dict[str, H5SourceItem]: - """ - Flatten all child elements of hdf element into a dict of :class:`.H5SourceItem` s - keyed by their path - - Args: - h5f (:class:`h5py.File` | :class:`h5py.Group`): HDF file or group to flatten! - """ - items = {} - - def _itemize(name: str, obj: h5py.Dataset | h5py.Group) -> None: - if skip in name: - return - - leaf = isinstance(obj, h5py.Dataset) or len(obj.keys()) == 0 - - if isinstance(obj, h5py.Dataset): - h5_type = "dataset" - elif isinstance(obj, h5py.Group): - h5_type = "group" - else: - raise ValueError(f"Object must be a dataset or group! {obj}") - - # get references in attrs and datasets to populate dependencies - # depends = get_references(obj) - - if not name.startswith("/"): - name = "/" + name - - attrs = dict(obj.attrs.items()) - - items[name] = H5SourceItem.model_construct( - path=name, - h5f_path=h5f.file.filename, - leaf=leaf, - # depends = depends, - h5_type=h5_type, - attrs=attrs, - namespace=attrs.get("namespace"), - neurodata_type=attrs.get("neurodata_type"), - ) - - h5f.visititems(_itemize) - # then add the root item - _itemize(h5f.name, h5f) - return items + if obj.shape == (): + # scalar + if isinstance(obj[()], h5py.h5r.Reference): + refs = [obj.file.get(obj[()]).name] + elif len(obj) > 0 and isinstance(obj[0], h5py.h5r.Reference): + # single-column + refs = [obj.file.get(ref).name for ref in obj[:]] + elif len(obj.dtype) > 1: + # "compound" datasets + refs = {} + for name in obj.dtype.names: + if isinstance(obj[name][0], h5py.h5r.Reference): + refs[name] = [obj.file.get(ref).name for ref in obj[name]] + return refs def get_references(obj: h5py.Dataset | h5py.Group) -> List[str]: @@ -851,60 +66,21 @@ def get_references(obj: h5py.Dataset | h5py.Group) -> List[str]: List[str]: List of paths that are referenced within this object """ # Find references in attrs - refs = [ref for ref in obj.attrs.values() if isinstance(ref, h5py.h5r.Reference)] + attr_refs = get_attr_references(obj) + dataset_refs = get_dataset_references(obj) - # For datasets, apply checks depending on shape of data. - if isinstance(obj, h5py.Dataset): - if obj.shape == (): - # scalar - if isinstance(obj[()], h5py.h5r.Reference): - refs.append(obj[()]) - elif isinstance(obj[0], h5py.h5r.Reference): - # single-column - refs.extend(obj[:].tolist()) - elif len(obj.dtype) > 1: - # "compound" datasets - for name in obj.dtype.names: - if isinstance(obj[name][0], h5py.h5r.Reference): - refs.extend(obj[name].tolist()) - - # dereference and get name of reference - if isinstance(obj, h5py.Dataset): - depends = list(set([obj.parent.get(i).name for i in refs])) + # flatten to list + refs = [ref for ref in attr_refs.values()] + if isinstance(dataset_refs, list): + refs.extend(dataset_refs) else: - depends = list(set([obj.get(i).name for i in refs])) - return depends - - -def resolve_references( - src: dict, completed: Dict[str, H5ReadResult] -) -> Tuple[dict, List[str], List[HDF5_Path]]: - """ - Recursively replace references to other completed items with their results - - """ - completes = [] - errors = [] - res = {} - for path, item in src.items(): - if isinstance(item, HDF5_Path): - other_item = completed.get(item) - if other_item is None: - errors.append(f"Couldn't find: {item}") - res[path] = other_item.result - completes.append(item) + for v in dataset_refs.values(): + refs.extend(v) - elif isinstance(item, dict): - inner_res, inner_error, inner_completes = resolve_references(item, completed) - res[path] = inner_res - errors.extend(inner_error) - completes.extend(inner_completes) - else: - res[path] = item - return res, errors, completes + return refs -def resolve_hardlink(obj: Union[h5py.Group, h5py.Dataset]) -> HDF5_Path: +def resolve_hardlink(obj: Union[h5py.Group, h5py.Dataset]) -> str: """ Unhelpfully, hardlinks are pretty challenging to detect with h5py, so we have to do extra work to check if an item is "real" or a hardlink to another item. @@ -916,4 +92,4 @@ def resolve_hardlink(obj: Union[h5py.Group, h5py.Dataset]) -> HDF5_Path: We basically dereference the object and return that path instead of the path given by the object's ``name`` """ - return HDF5_Path(obj.file[obj.ref].name) + return obj.file[obj.ref].name diff --git a/nwb_linkml/src/nwb_linkml/providers/linkml.py b/nwb_linkml/src/nwb_linkml/providers/linkml.py index 4af2bec8..fe8dec54 100644 --- a/nwb_linkml/src/nwb_linkml/providers/linkml.py +++ b/nwb_linkml/src/nwb_linkml/providers/linkml.py @@ -127,6 +127,7 @@ def build_from_dicts( for schema_needs in adapter.needed_imports.values(): for needed in schema_needs: adapter.imported.append(ns_adapters[needed]) + adapter.populate_imports() # then do the build res = {} diff --git a/nwb_linkml/src/nwb_linkml/providers/provider.py b/nwb_linkml/src/nwb_linkml/providers/provider.py index 87f65675..ff349afb 100644 --- a/nwb_linkml/src/nwb_linkml/providers/provider.py +++ b/nwb_linkml/src/nwb_linkml/providers/provider.py @@ -97,9 +97,9 @@ def namespace_path( module_path = Path(importlib.util.find_spec("nwb_models").origin).parent if self.PROVIDES == "linkml": - namespace_path = module_path / "schema" / "linkml" / namespace + namespace_path = module_path / "schema" / "linkml" / namespace_module elif self.PROVIDES == "pydantic": - namespace_path = module_path / "models" / "pydantic" / namespace + namespace_path = module_path / "models" / "pydantic" / namespace_module if version is not None: version_path = namespace_path / version_module_case(version) diff --git a/nwb_linkml/src/nwb_linkml/providers/pydantic.py b/nwb_linkml/src/nwb_linkml/providers/pydantic.py index 5d5975cd..c44c85c6 100644 --- a/nwb_linkml/src/nwb_linkml/providers/pydantic.py +++ b/nwb_linkml/src/nwb_linkml/providers/pydantic.py @@ -278,7 +278,7 @@ def module_name(self, namespace: str, version: str) -> str: nwb_models.models.pydantic.{namespace}.{version} """ name_pieces = [ - "nwb_linkml", + "nwb_models", "models", "pydantic", module_case(namespace), diff --git a/nwb_linkml/src/nwb_linkml/providers/schema.py b/nwb_linkml/src/nwb_linkml/providers/schema.py index adadb002..7555f93f 100644 --- a/nwb_linkml/src/nwb_linkml/providers/schema.py +++ b/nwb_linkml/src/nwb_linkml/providers/schema.py @@ -131,7 +131,7 @@ def build( results = {} for ns, ns_result in linkml_res.items(): results[ns] = pydantic_provider.build( - ns_result["namespace"], versions=self.versions, **pydantic_kwargs + ns_result.namespace, versions=self.versions, **pydantic_kwargs ) return results diff --git a/nwb_linkml/src/nwb_linkml/types/hdf5.py b/nwb_linkml/src/nwb_linkml/types/hdf5.py deleted file mode 100644 index 9f745763..00000000 --- a/nwb_linkml/src/nwb_linkml/types/hdf5.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Types used with hdf5 io -""" - -from typing import Any - -from pydantic import GetCoreSchemaHandler -from pydantic_core import CoreSchema, core_schema - - -class HDF5_Path(str): - """ - Trivial subclass of string to indicate that it is a reference to a location within an HDF5 file - """ - - @classmethod - def __get_pydantic_core_schema__( - cls, source_type: Any, handler: GetCoreSchemaHandler - ) -> CoreSchema: - return core_schema.no_info_after_validator_function(cls, handler(str)) diff --git a/nwb_linkml/tests/conftest.py b/nwb_linkml/tests/conftest.py index 6133148e..7bb36322 100644 --- a/nwb_linkml/tests/conftest.py +++ b/nwb_linkml/tests/conftest.py @@ -9,10 +9,16 @@ def pytest_addoption(parser): + parser.addoption( + "--clean", + action="store_true", + default=False, + help="Don't reuse cached resources like cloned git repos or generated files", + ) parser.addoption( "--with-output", action="store_true", - help="dump output in compliance test for richer debugging information", + help="keep test outputs for richer debugging information", ) parser.addoption( "--without-cache", action="store_true", help="Don't use a sqlite cache for network requests" diff --git a/nwb_linkml/tests/data/aibs.nwb b/nwb_linkml/tests/data/aibs.nwb index 6000e09d..1380c551 100644 Binary files a/nwb_linkml/tests/data/aibs.nwb and b/nwb_linkml/tests/data/aibs.nwb differ diff --git a/nwb_linkml/tests/data/aibs_ecephys.nwb b/nwb_linkml/tests/data/aibs_ecephys.nwb index 4a5ad9ca..cc150383 100644 Binary files a/nwb_linkml/tests/data/aibs_ecephys.nwb and b/nwb_linkml/tests/data/aibs_ecephys.nwb differ diff --git a/nwb_linkml/tests/data/test_nwb.yaml b/nwb_linkml/tests/data/test_nwb.yaml new file mode 100644 index 00000000..defa1187 --- /dev/null +++ b/nwb_linkml/tests/data/test_nwb.yaml @@ -0,0 +1,61 @@ +# manually transcribed target version of nwb-linkml dataset +# matching the one created by fixtures.py:nwb_file +meta: + id: my_dataset + + prefixes: + nwbfile: + - path: "test_nwb.nwb" + - hash: "blake2b:blahblahblahblah" + + imports: + core: + as: nwb + version: "2.7.0" + from: + - pypi: + package: nwb-models + hdmf-common: + as: hdmf + version: "1.8.0" + from: + - pypi: + package: nwb-models + +extracellular_ephys: &ecephys + electrodes: + group: + - @shank0 + - @shank0 + - @shank0 + - @shank1 + - # etc. + shank0: + device: @general.devices.array + shank1: + device: @general.devices.array + # etc. + +data: !nwb.NWBFile + file_create_date: [ 2024-01-01 ] + identifier: "1111-1111-1111-1111" + session_description: All that you touch, you change. + session_start_time: 2024-01-01T01:01:01 + general: + devices: + - Heka ITC-1600: + - Microscope: + description: My two-photon microscope + manufacturer: The best microscope manufacturer + - array: + description: old reliable + manufacturer: diy + extracellular_ephys: nwbfile:/general/extracellular_ephys + experiment_description: All that you change, changes you. + experimenter: [ "Lauren Oya Olamina" ] + institution: Earthseed Research Institute + keywords: + - behavior + - belief + related_publications: doi:10.1016/j.neuron.2016.12.011 + diff --git a/nwb_linkml/tests/data/test_nwb_condensed_sketch.yaml b/nwb_linkml/tests/data/test_nwb_condensed_sketch.yaml new file mode 100644 index 00000000..d9ca7c7c --- /dev/null +++ b/nwb_linkml/tests/data/test_nwb_condensed_sketch.yaml @@ -0,0 +1,76 @@ +# Sketch of a condensed expression syntax for creation with nwb-linkml +# just a sketch! keeping here for continued work but currently unused. +--- +id: my_dataset + +prefixes: + nwbfile: + - path: "test_nwb.nwb" + - hash: "blake2b:blahblahblahblah" + +imports: + core: + as: nwb + version: "2.7.0" + from: + - pypi: + package: nwb-models + hdmf-common: + as: hdmf + version: "1.8.0" + from: + - pypi: + package: nwb-models +--- + +extracellular_ephys: &ecephys + electrodes: + group: + - @shank{{i}} + - @shank{{i}} + - @shank{{i}} + # could have expression here like { range(3) } => i + # - ... { range(3) } => i + # or blank ... implies use expression from outer scope + - ... + shank{{i}}: + device: @general.devices.array + ...: { range(3) } => i + +# expands to +extracellular_ephys: + electrodes: + group: + - @shank0 + - @shank0 + - @shank0 + - @shank1 + - # etc. + shank0: + device: @general.devices.array + shank1: + device: @general.devices.array + # etc. + +data: !{{ nwb.NWBFile }} <== :nwbfile + file_create_date: [ 2024-01-01 ] + identifier: "1111-1111-1111-1111" + session_description: All that you touch, you change. + session_start_time: 2024-01-01T01:01:01 + general: + devices: + - Heka ITC-1600: + - Microscope: + - array: + description: old reliable + manufacturer: diy + extracellular_ephys: *ecephys + + experiment_description: All that you change, changes you. + experimenter: [ "Lauren Oya Olamina" ] + institution: Earthseed Research Institute + keywords: + - behavior + - belief + related_publications: doi:10.1016/j.neuron.2016.12.011 + diff --git a/nwb_linkml/tests/fixtures/__init__.py b/nwb_linkml/tests/fixtures/__init__.py new file mode 100644 index 00000000..f135929a --- /dev/null +++ b/nwb_linkml/tests/fixtures/__init__.py @@ -0,0 +1,29 @@ +from .nwb import nwb_file, nwb_file_base +from .paths import data_dir, tmp_output_dir, tmp_output_dir_func, tmp_output_dir_mod +from .schema import ( + NWBSchemaTest, + TestSchemas, + linkml_schema, + linkml_schema_bare, + nwb_core_fixture, + nwb_core_linkml, + nwb_core_module, + nwb_schema, +) + +__all__ = [ + "NWBSchemaTest", + "TestSchemas", + "data_dir", + "linkml_schema", + "linkml_schema_bare", + "nwb_core_fixture", + "nwb_core_linkml", + "nwb_core_module", + "nwb_file", + "nwb_file_base", + "nwb_schema", + "tmp_output_dir", + "tmp_output_dir_func", + "tmp_output_dir_mod", +] diff --git a/nwb_linkml/tests/fixtures/nwb.py b/nwb_linkml/tests/fixtures/nwb.py new file mode 100644 index 00000000..c878c74d --- /dev/null +++ b/nwb_linkml/tests/fixtures/nwb.py @@ -0,0 +1,477 @@ +from datetime import datetime +from itertools import product +from pathlib import Path + +import numpy as np +import pytest +from hdmf.common import DynamicTable, VectorData +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from pynwb.base import TimeSeriesReference, TimeSeriesReferenceVectorData +from pynwb.behavior import Position, SpatialSeries +from pynwb.ecephys import LFP, ElectricalSeries +from pynwb.file import Subject +from pynwb.icephys import VoltageClampSeries, VoltageClampStimulusSeries +from pynwb.image import ImageSeries +from pynwb.ophys import ( + CorrectedImageStack, + Fluorescence, + ImageSegmentation, + MotionCorrection, + OnePhotonSeries, + OpticalChannel, + RoiResponseSeries, + TwoPhotonSeries, +) + + +@pytest.fixture(scope="session") +def nwb_file_base() -> NWBFile: + nwbfile = NWBFile( + session_description="All that you touch, you change.", # required + identifier="1111-1111-1111-1111", # required + session_start_time=datetime(year=2024, month=1, day=1), # required + session_id="session_1234", # optional + experimenter=[ + "Lauren Oya Olamina", + ], # optional + institution="Earthseed Research Institute", # optional + experiment_description="All that you change, changes you.", # optional + keywords=["behavior", "belief"], # optional + related_publications="doi:10.1016/j.neuron.2016.12.011", # optional + ) + subject = Subject( + subject_id="001", + age="P90D", + description="mouse 5", + species="Mus musculus", + sex="M", + ) + nwbfile.subject = subject + return nwbfile + + +def _nwb_timeseries(nwbfile: NWBFile) -> NWBFile: + data = np.arange(100, 200, 10) + timestamps = np.arange(10.0) + time_series_with_timestamps = TimeSeries( + name="test_timeseries", + description="an example time series", + data=data, + unit="m", + timestamps=timestamps, + ) + nwbfile.add_acquisition(time_series_with_timestamps) + return nwbfile + + +def _nwb_position(nwbfile: NWBFile) -> NWBFile: + position_data = np.array([np.linspace(0, 10, 50), np.linspace(0, 8, 50)]).T + position_timestamps = np.linspace(0, 50).astype(float) / 200 + + spatial_series_obj = SpatialSeries( + name="SpatialSeries", + description="(x,y) position in open field", + data=position_data, + timestamps=position_timestamps, + reference_frame="(0,0) is bottom left corner", + ) + # name is set to "Position" by default + position_obj = Position(spatial_series=spatial_series_obj) + behavior_module = nwbfile.create_processing_module( + name="behavior", description="processed behavioral data" + ) + behavior_module.add(position_obj) + + nwbfile.add_trial_column( + name="correct", + description="whether the trial was correct", + ) + nwbfile.add_trial(start_time=1.0, stop_time=5.0, correct=True) + nwbfile.add_trial(start_time=6.0, stop_time=10.0, correct=False) + return nwbfile + + +def _nwb_ecephys(nwbfile: NWBFile) -> NWBFile: + """ + Extracellular Ephys + https://pynwb.readthedocs.io/en/latest/tutorials/domain/ecephys.html + """ + generator = np.random.default_rng() + device = nwbfile.create_device(name="array", description="old reliable", manufacturer="diy") + nwbfile.add_electrode_column(name="label", description="label of electrode") + + nshanks = 4 + nchannels_per_shank = 3 + electrode_counter = 0 + + for ishank in range(nshanks): + # create an electrode group for this shank + electrode_group = nwbfile.create_electrode_group( + name=f"shank{ishank}", + description=f"electrode group for shank {ishank}", + device=device, + location="brain area", + ) + # add electrodes to the electrode table + for ielec in range(nchannels_per_shank): + nwbfile.add_electrode( + group=electrode_group, + label=f"shank{ishank}elec{ielec}", + location="brain area", + ) + electrode_counter += 1 + all_table_region = nwbfile.create_electrode_table_region( + region=list(range(electrode_counter)), # reference row indices 0 to N-1 + description="all electrodes", + ) + raw_data = generator.standard_normal((50, 12)) + raw_electrical_series = ElectricalSeries( + name="ElectricalSeries", + description="Raw acquisition traces", + data=raw_data, + electrodes=all_table_region, + starting_time=0.0, + # timestamp of the first sample in seconds relative to the session start time + rate=20000.0, # in Hz + ) + nwbfile.add_acquisition(raw_electrical_series) + + # -------------------------------------------------- + # LFP + # -------------------------------------------------- + generator = np.random.default_rng() + lfp_data = generator.standard_normal((50, 12)) + lfp_electrical_series = ElectricalSeries( + name="ElectricalSeries", + description="LFP data", + data=lfp_data, + electrodes=all_table_region, + starting_time=0.0, + rate=200.0, + ) + lfp = LFP(electrical_series=lfp_electrical_series) + ecephys_module = nwbfile.create_processing_module( + name="ecephys", description="processed extracellular electrophysiology data" + ) + ecephys_module.add(lfp) + + return nwbfile + + +def _nwb_units(nwbfile: NWBFile) -> NWBFile: + generator = np.random.default_rng() + # Spike Times + nwbfile.add_unit_column(name="quality", description="sorting quality") + firing_rate = 20 + n_units = 10 + res = 1000 + duration = 20 + for _ in range(n_units): + spike_times = np.where(generator.random(res * duration) < (firing_rate / res))[0] / res + nwbfile.add_unit(spike_times=spike_times, quality="good") + return nwbfile + + +def _nwb_icephys(nwbfile: NWBFile) -> NWBFile: + device = nwbfile.create_device(name="Heka ITC-1600") + electrode = nwbfile.create_icephys_electrode( + name="elec0", description="a mock intracellular electrode", device=device + ) + stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=electrode, + gain=0.02, + sweep_number=np.uint64(15), + ) + + # Create and icephys response + response = VoltageClampSeries( + name="vcs", + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15), + ) + # we can also add stimulus template data as follows + rowindex = nwbfile.add_intracellular_recording( + electrode=electrode, stimulus=stimulus, response=response, id=10 + ) + + rowindex2 = nwbfile.add_intracellular_recording( + electrode=electrode, + stimulus=stimulus, + stimulus_start_index=1, + stimulus_index_count=3, + response=response, + response_start_index=2, + response_index_count=3, + id=11, + ) + rowindex3 = nwbfile.add_intracellular_recording(electrode=electrode, response=response, id=12) + + nwbfile.intracellular_recordings.add_column( + name="recording_tag", + data=["A1", "A2", "A3"], + description="String with a recording tag", + ) + location_column = VectorData( + name="location", + data=["Mordor", "Gondor", "Rohan"], + description="Recording location in Middle Earth", + ) + + lab_category = DynamicTable( + name="recording_lab_data", + description="category table for lab-specific recording metadata", + colnames=[ + "location", + ], + columns=[ + location_column, + ], + ) + # Add the table as a new category to our intracellular_recordings + nwbfile.intracellular_recordings.add_category(category=lab_category) + nwbfile.intracellular_recordings.add_column( + name="voltage_threshold", + data=[0.1, 0.12, 0.13], + description="Just an example column on the electrodes category table", + category="electrodes", + ) + stimulus_template = VoltageClampStimulusSeries( + name="ccst", + data=[0, 1, 2, 3, 4], + starting_time=0.0, + rate=10e3, + electrode=electrode, + gain=0.02, + ) + nwbfile.add_stimulus_template(stimulus_template) + + nwbfile.intracellular_recordings.add_column( + name="stimulus_template", + data=[ + TimeSeriesReference(0, 5, stimulus_template), + # (start_index, index_count, stimulus_template) + TimeSeriesReference(1, 3, stimulus_template), + TimeSeriesReference.empty(stimulus_template), + ], + # if there was no data for that recording, use empty reference + description=( + "Column storing the reference to the stimulus template for the recording (rows)." + ), + category="stimuli", + col_cls=TimeSeriesReferenceVectorData, + ) + + icephys_simultaneous_recordings = nwbfile.get_icephys_simultaneous_recordings() + icephys_simultaneous_recordings.add_column( + name="simultaneous_recording_tag", + description="A custom tag for simultaneous_recordings", + ) + simultaneous_index = nwbfile.add_icephys_simultaneous_recording( + recordings=[rowindex, rowindex2, rowindex3], + id=12, + simultaneous_recording_tag="LabTag1", + ) + repetition_index = nwbfile.add_icephys_repetition( + sequential_recordings=[simultaneous_index], id=17 + ) + nwbfile.add_icephys_experimental_condition(repetitions=[repetition_index], id=19) + nwbfile.icephys_experimental_conditions.add_column( + name="tag", + data=np.arange(1), + description="integer tag for a experimental condition", + ) + return nwbfile + + +def _nwb_ca_imaging(nwbfile: NWBFile) -> NWBFile: + """ + Calcium Imaging + https://pynwb.readthedocs.io/en/latest/tutorials/domain/ophys.html + """ + generator = np.random.default_rng() + device = nwbfile.create_device( + name="Microscope", + description="My two-photon microscope", + manufacturer="The best microscope manufacturer", + ) + optical_channel = OpticalChannel( + name="OpticalChannel", + description="an optical channel", + emission_lambda=500.0, + ) + imaging_plane = nwbfile.create_imaging_plane( + name="ImagingPlane", + optical_channel=optical_channel, + imaging_rate=30.0, + description="a very interesting part of the brain", + device=device, + excitation_lambda=600.0, + indicator="GFP", + location="V1", + grid_spacing=[0.01, 0.01], + grid_spacing_unit="meters", + origin_coords=[1.0, 2.0, 3.0], + origin_coords_unit="meters", + ) + one_p_series = OnePhotonSeries( + name="OnePhotonSeries", + description="Raw 1p data", + data=np.ones((1000, 100, 100)), + imaging_plane=imaging_plane, + rate=1.0, + unit="normalized amplitude", + ) + nwbfile.add_acquisition(one_p_series) + two_p_series = TwoPhotonSeries( + name="TwoPhotonSeries", + description="Raw 2p data", + data=np.ones((1000, 100, 100)), + imaging_plane=imaging_plane, + rate=1.0, + unit="normalized amplitude", + ) + + nwbfile.add_acquisition(two_p_series) + + corrected = ImageSeries( + name="corrected", # this must be named "corrected" + description="A motion corrected image stack", + data=np.ones((1000, 100, 100)), + unit="na", + format="raw", + starting_time=0.0, + rate=1.0, + ) + + xy_translation = TimeSeries( + name="xy_translation", + description="x,y translation in pixels", + data=np.ones((1000, 2)), + unit="pixels", + starting_time=0.0, + rate=1.0, + ) + + corrected_image_stack = CorrectedImageStack( + corrected=corrected, + original=one_p_series, + xy_translation=xy_translation, + ) + + motion_correction = MotionCorrection(corrected_image_stacks=[corrected_image_stack]) + + ophys_module = nwbfile.create_processing_module( + name="ophys", description="optical physiology processed data" + ) + + ophys_module.add(motion_correction) + + img_seg = ImageSegmentation() + + ps = img_seg.create_plane_segmentation( + name="PlaneSegmentation", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + ophys_module.add(img_seg) + + for _ in range(30): + image_mask = np.zeros((100, 100)) + + # randomly generate example image masks + x = generator.integers(0, 95) + y = generator.integers(0, 95) + image_mask[x : x + 5, y : y + 5] = 1 + + # add image mask to plane segmentation + ps.add_roi(image_mask=image_mask) + + ps2 = img_seg.create_plane_segmentation( + name="PlaneSegmentation2", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + for _ in range(30): + # randomly generate example starting points for region + x = generator.integers(0, 95) + y = generator.integers(0, 95) + + # define an example 4 x 3 region of pixels of weight '1' + pixel_mask = [(ix, iy, 1) for ix in range(x, x + 4) for iy in range(y, y + 3)] + + # add pixel mask to plane segmentation + ps2.add_roi(pixel_mask=pixel_mask) + + ps3 = img_seg.create_plane_segmentation( + name="PlaneSegmentation3", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + for _ in range(30): + # randomly generate example starting points for region + x = generator.integers(0, 95) + y = generator.integers(0, 95) + z = generator.integers(0, 15) + + # define an example 4 x 3 x 2 voxel region of weight '0.5' + voxel_mask = [] + for ix, iy, iz in product(range(x, x + 4), range(y, y + 3), range(z, z + 2)): + voxel_mask.append((ix, iy, iz, 0.5)) + + # add voxel mask to plane segmentation + ps3.add_roi(voxel_mask=voxel_mask) + rt_region = ps.create_roi_table_region(region=[0, 1], description="the first of two ROIs") + roi_resp_series = RoiResponseSeries( + name="RoiResponseSeries", + description="Fluorescence responses for two ROIs", + data=np.ones((50, 2)), # 50 samples, 2 ROIs + rois=rt_region, + unit="lumens", + rate=30.0, + ) + fl = Fluorescence(roi_response_series=roi_resp_series) + ophys_module.add(fl) + return nwbfile + + +@pytest.fixture(scope="session") +def nwb_file(tmp_output_dir, nwb_file_base, request: pytest.FixtureRequest) -> Path: + """ + NWB File created with pynwb that uses all the weird language features + + Borrowing code from pynwb docs in one humonogous fixture function + since there's not really a reason to + """ + nwb_path = tmp_output_dir / "test_nwb.nwb" + if nwb_path.exists() and not request.config.getoption("--clean"): + return nwb_path + + nwbfile = nwb_file_base + nwbfile = _nwb_timeseries(nwbfile) + nwbfile = _nwb_position(nwbfile) + nwbfile = _nwb_ecephys(nwbfile) + nwbfile = _nwb_units(nwbfile) + nwbfile = _nwb_icephys(nwbfile) + + with NWBHDF5IO(nwb_path, "w") as io: + io.write(nwbfile) + + return nwb_path diff --git a/nwb_linkml/tests/fixtures/paths.py b/nwb_linkml/tests/fixtures/paths.py new file mode 100644 index 00000000..f2d0e1e9 --- /dev/null +++ b/nwb_linkml/tests/fixtures/paths.py @@ -0,0 +1,63 @@ +import shutil +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session") +def tmp_output_dir(request: pytest.FixtureRequest) -> Path: + path = Path(__file__).parents[1].resolve() / "__tmp__" + if path.exists(): + if request.config.getoption("--clean"): + shutil.rmtree(path) + else: + for subdir in path.iterdir(): + if subdir.name == "git": + # don't wipe out git repos every time, they don't rly change + continue + elif ( + subdir.is_file() + and subdir.parent != path + or subdir.is_file() + and subdir.suffix == ".nwb" + ): + continue + elif subdir.is_file(): + subdir.unlink(missing_ok=True) + else: + shutil.rmtree(str(subdir)) + path.mkdir(exist_ok=True) + + return path + + +@pytest.fixture(scope="function") +def tmp_output_dir_func(tmp_output_dir) -> Path: + """ + tmp output dir that gets cleared between every function + cleans at the start rather than at cleanup in case the output is to be inspected + """ + subpath = tmp_output_dir / "__tmpfunc__" + if subpath.exists(): + shutil.rmtree(str(subpath)) + subpath.mkdir() + return subpath + + +@pytest.fixture(scope="module") +def tmp_output_dir_mod(tmp_output_dir) -> Path: + """ + tmp output dir that gets cleared between every function + cleans at the start rather than at cleanup in case the output is to be inspected + """ + subpath = tmp_output_dir / "__tmpmod__" + if subpath.exists(): + shutil.rmtree(str(subpath)) + subpath.mkdir() + return subpath + + +@pytest.fixture(scope="session") +def data_dir() -> Path: + path = Path(__file__).parents[1].resolve() / "data" + return path diff --git a/nwb_linkml/tests/fixtures.py b/nwb_linkml/tests/fixtures/schema.py similarity index 83% rename from nwb_linkml/tests/fixtures.py rename to nwb_linkml/tests/fixtures/schema.py index a38e3e00..788d12b3 100644 --- a/nwb_linkml/tests/fixtures.py +++ b/nwb_linkml/tests/fixtures/schema.py @@ -1,4 +1,3 @@ -import shutil from dataclasses import dataclass, field from pathlib import Path from types import ModuleType @@ -14,70 +13,12 @@ TypeDefinition, ) -from nwb_linkml.adapters.namespaces import NamespacesAdapter +from nwb_linkml.adapters import NamespacesAdapter from nwb_linkml.io import schema as io from nwb_linkml.providers import LinkMLProvider, PydanticProvider from nwb_linkml.providers.linkml import LinkMLSchemaBuild from nwb_schema_language import Attribute, Dataset, Group -__all__ = [ - "NWBSchemaTest", - "TestSchemas", - "data_dir", - "linkml_schema", - "linkml_schema_bare", - "nwb_core_fixture", - "nwb_schema", - "tmp_output_dir", - "tmp_output_dir_func", - "tmp_output_dir_mod", -] - - -@pytest.fixture(scope="session") -def tmp_output_dir() -> Path: - path = Path(__file__).parent.resolve() / "__tmp__" - if path.exists(): - for subdir in path.iterdir(): - if subdir.name == "git": - # don't wipe out git repos every time, they don't rly change - continue - elif subdir.is_file() and subdir.parent != path: - continue - elif subdir.is_file(): - subdir.unlink(missing_ok=True) - else: - shutil.rmtree(str(subdir)) - path.mkdir(exist_ok=True) - - return path - - -@pytest.fixture(scope="function") -def tmp_output_dir_func(tmp_output_dir) -> Path: - """ - tmp output dir that gets cleared between every function - cleans at the start rather than at cleanup in case the output is to be inspected - """ - subpath = tmp_output_dir / "__tmpfunc__" - if subpath.exists(): - shutil.rmtree(str(subpath)) - subpath.mkdir() - return subpath - - -@pytest.fixture(scope="module") -def tmp_output_dir_mod(tmp_output_dir) -> Path: - """ - tmp output dir that gets cleared between every function - cleans at the start rather than at cleanup in case the output is to be inspected - """ - subpath = tmp_output_dir / "__tmpmod__" - if subpath.exists(): - shutil.rmtree(str(subpath)) - subpath.mkdir() - return subpath - @pytest.fixture(scope="session", params=[{"core_version": "2.7.0", "hdmf_version": "1.8.0"}]) def nwb_core_fixture(request) -> NamespacesAdapter: @@ -108,12 +49,6 @@ def nwb_core_module(nwb_core_linkml: LinkMLSchemaBuild, tmp_output_dir) -> Modul return mod -@pytest.fixture(scope="session") -def data_dir() -> Path: - path = Path(__file__).parent.resolve() / "data" - return path - - @dataclass class TestSchemas: __test__ = False diff --git a/nwb_linkml/tests/test_adapters/test_adapter_classes.py b/nwb_linkml/tests/test_adapters/test_adapter_classes.py index ee6e7f66..48fe6036 100644 --- a/nwb_linkml/tests/test_adapters/test_adapter_classes.py +++ b/nwb_linkml/tests/test_adapters/test_adapter_classes.py @@ -151,7 +151,7 @@ def test_name_slot(): assert slot.name == "name" assert slot.required assert slot.range == "string" - assert slot.identifier is None + assert slot.identifier assert slot.ifabsent is None assert slot.equals_string is None @@ -160,7 +160,7 @@ def test_name_slot(): assert slot.name == "name" assert slot.required assert slot.range == "string" - assert slot.identifier is None + assert slot.identifier assert slot.ifabsent == "string(FixedName)" assert slot.equals_string == "FixedName" diff --git a/nwb_linkml/tests/test_generators/test_generator_pydantic.py b/nwb_linkml/tests/test_generators/test_generator_pydantic.py index fdab1477..12021f4f 100644 --- a/nwb_linkml/tests/test_generators/test_generator_pydantic.py +++ b/nwb_linkml/tests/test_generators/test_generator_pydantic.py @@ -5,6 +5,8 @@ because it's tested in the base linkml package. """ +# ruff: noqa: F821 - until the tests here settle down + import re import sys import typing @@ -16,7 +18,7 @@ from numpydantic.ndarray import NDArrayMeta from pydantic import BaseModel -from nwb_linkml.generators.pydantic import NWBPydanticGenerator, compile_python +from nwb_linkml.generators.pydantic import NWBPydanticGenerator from ..fixtures import ( TestSchemas, diff --git a/nwb_linkml/tests/test_includes/test_hdmf.py b/nwb_linkml/tests/test_includes/test_hdmf.py index 920e8b4d..a8b14b7b 100644 --- a/nwb_linkml/tests/test_includes/test_hdmf.py +++ b/nwb_linkml/tests/test_includes/test_hdmf.py @@ -284,14 +284,14 @@ class MyDT(DynamicTableMixin): "existing_col": np.arange(10), "new_col_1": hdmf.VectorData(value=np.arange(11)), } - with pytest.raises(ValidationError, match="Columns are not of equal length"): + with pytest.raises(ValidationError, match="columns are not of equal length"): _ = MyDT(**cols) cols = { "existing_col": np.arange(11), "new_col_1": hdmf.VectorData(value=np.arange(10)), } - with pytest.raises(ValidationError, match="Columns are not of equal length"): + with pytest.raises(ValidationError, match="columns are not of equal length"): _ = MyDT(**cols) # wrong lengths are fine as long as the index is good @@ -308,7 +308,7 @@ class MyDT(DynamicTableMixin): "new_col_1": hdmf.VectorData(value=np.arange(100)), "new_col_1_index": hdmf.VectorIndex(value=np.arange(0, 100, 5) + 5), } - with pytest.raises(ValidationError, match="Columns are not of equal length"): + with pytest.raises(ValidationError, match="columns are not of equal length"): _ = MyDT(**cols) @@ -344,7 +344,7 @@ def test_vectordata_indexing(): """ n_rows = 50 value_array, index_array = _ragged_array(n_rows) - value_array = np.concat(value_array) + value_array = np.concatenate(value_array) data = hdmf.VectorData(value=value_array) @@ -551,13 +551,13 @@ def test_aligned_dynamictable_indexing(aligned_table): row.columns == pd.MultiIndex.from_tuples( [ - ("table1", "index"), + ("table1", "id"), ("table1", "col1"), ("table1", "col2"), - ("table2", "index"), + ("table2", "id"), ("table2", "col3"), ("table2", "col4"), - ("table3", "index"), + ("table3", "id"), ("table3", "col5"), ("table3", "col6"), ] @@ -592,7 +592,7 @@ def test_mixed_aligned_dynamictable(aligned_table): AlignedTable, cols = aligned_table value_array, index_array = _ragged_array(10) - value_array = np.concat(value_array) + value_array = np.concatenate(value_array) data = hdmf.VectorData(value=value_array) index = hdmf.VectorIndex(value=index_array) @@ -754,11 +754,11 @@ def test_aligned_dynamictable_ictable(intracellular_recordings_table): rows.columns == pd.MultiIndex.from_tuples( [ - ("electrodes", "index"), + ("electrodes", "id"), ("electrodes", "electrode"), - ("stimuli", "index"), + ("stimuli", "id"), ("stimuli", "stimulus"), - ("responses", "index"), + ("responses", "id"), ("responses", "response"), ] ) diff --git a/nwb_linkml/tests/test_io/test_io_hdf5.py b/nwb_linkml/tests/test_io/test_io_hdf5.py index c64cf486..4222a2cb 100644 --- a/nwb_linkml/tests/test_io/test_io_hdf5.py +++ b/nwb_linkml/tests/test_io/test_io_hdf5.py @@ -1,10 +1,10 @@ -import pdb - import h5py +import networkx as nx import numpy as np import pytest -from nwb_linkml.io.hdf5 import HDF5IO, truncate_file +from nwb_linkml.io.hdf5 import HDF5IO, filter_dependency_graph, hdf_dependency_graph, truncate_file +from nwb_linkml.maps.hdf5 import resolve_hardlink @pytest.mark.skip() @@ -13,7 +13,7 @@ def test_hdf_read(data_dir, dset): NWBFILE = data_dir / dset io = HDF5IO(path=NWBFILE) # the test for now is just whether we can read it lol - model = io.read() + _ = io.read() def test_truncate_file(tmp_output_dir): @@ -86,15 +86,60 @@ def test_truncate_file(tmp_output_dir): assert target_h5f["data"]["dataset_contig"].attrs["anattr"] == 1 -@pytest.mark.skip() -def test_flatten_hdf(): - from nwb_linkml.maps.hdf5 import flatten_hdf - - path = "/Users/jonny/Dropbox/lab/p2p_ld/data/nwb/sub-738651046_ses-760693773.nwb" - import h5py - - h5f = h5py.File(path) - flat = flatten_hdf(h5f) - assert not any(["specifications" in v.path for v in flat.values()]) - pdb.set_trace() - raise NotImplementedError("Just a stub for local testing for now, finish me!") +def test_dependencies_hardlink(nwb_file): + """ + Test that hardlinks are resolved (eg. from /processing/ecephys/LFP/ElectricalSeries/electrodes + to /acquisition/ElectricalSeries/electrodes + Args: + nwb_file: + + Returns: + + """ + parent = "/processing/ecephys/LFP/ElectricalSeries" + source = "/processing/ecephys/LFP/ElectricalSeries/electrodes" + target = "/acquisition/ElectricalSeries/electrodes" + + # assert that the hardlink exists in the test file + with h5py.File(str(nwb_file), "r") as h5f: + node = h5f.get(source) + linked_node = resolve_hardlink(node) + assert linked_node == target + + graph = hdf_dependency_graph(nwb_file) + # the parent should link to the target as a child + assert (parent, target) in graph.edges([parent]) + assert graph.edges[parent, target]["label"] == "child" + + +@pytest.mark.dev +def test_dependency_graph_images(nwb_file, tmp_output_dir): + """ + Generate images of the dependency graph + """ + graph = hdf_dependency_graph(nwb_file) + A_unfiltered = nx.nx_agraph.to_agraph(graph) + A_unfiltered.draw(tmp_output_dir / "test_nwb_unfiltered.png", prog="dot") + graph = filter_dependency_graph(graph) + A_filtered = nx.nx_agraph.to_agraph(graph) + A_filtered.draw(tmp_output_dir / "test_nwb_filtered.png", prog="dot") + + +@pytest.mark.parametrize( + "dset", + [ + {"name": "aibs.nwb", "source": "sub-738651046_ses-760693773.nwb"}, + { + "name": "aibs_ecephys.nwb", + "source": "sub-738651046_ses-760693773_probe-769322820_ecephys.nwb", + }, + ], +) +@pytest.mark.dev +def test_make_truncated_datasets(tmp_output_dir, data_dir, dset): + input_file = tmp_output_dir / dset["source"] + output_file = data_dir / dset["name"] + if not input_file.exists(): + return + + truncate_file(input_file, output_file, 10) diff --git a/nwb_linkml/tests/test_io/test_io_nwb.py b/nwb_linkml/tests/test_io/test_io_nwb.py new file mode 100644 index 00000000..1ad51edf --- /dev/null +++ b/nwb_linkml/tests/test_io/test_io_nwb.py @@ -0,0 +1,110 @@ +""" +Placeholder test module to test reading from pynwb-generated NWB file +""" + +from datetime import datetime + +import numpy as np +import pandas as pd +import pytest +from numpydantic.interface.hdf5 import H5Proxy +from pydantic import BaseModel +from pynwb import NWBHDF5IO +from pynwb import NWBFile as PyNWBFile + +from nwb_linkml.io.hdf5 import HDF5IO +from nwb_models.models import NWBFile + + +def test_read_from_nwbfile(nwb_file): + """ + Read data from a pynwb HDF5 NWB file + + Placeholder that just ensures that reads work and all pydantic models validate, + testing of correctness of read will happen elsewhere. + """ + res = HDF5IO(nwb_file).read() + + +@pytest.fixture(scope="module") +def read_nwbfile(nwb_file) -> NWBFile: + res = HDF5IO(nwb_file).read() + return res + + +@pytest.fixture(scope="module") +def read_pynwb(nwb_file) -> PyNWBFile: + nwbf = NWBHDF5IO(nwb_file, "r") + res = nwbf.read() + yield res + nwbf.close() + + +def _compare_attrs(model: BaseModel, pymodel: object): + for field, value in model.model_dump().items(): + if isinstance(value, (dict, H5Proxy)): + continue + if hasattr(pymodel, field): + pynwb_val = getattr(pymodel, field) + if isinstance(pynwb_val, list): + if isinstance(pynwb_val[0], datetime): + # need to normalize UTC numpy.datetime64 with datetime with tz + continue + assert all([val == pval for val, pval in zip(value, pynwb_val)]) + else: + if not pynwb_val: + # pynwb instantiates some stuff as empty dicts where we use ``None`` + assert bool(pynwb_val) == bool(value) + else: + assert value == pynwb_val + + +def test_nwbfile_base(read_nwbfile, read_pynwb): + """ + Base attributes on top-level nwbfile are correct + """ + _compare_attrs(read_nwbfile, read_pynwb) + + +def test_timeseries(read_nwbfile, read_pynwb): + py_acq = read_pynwb.get_acquisition("test_timeseries") + acq = read_nwbfile.acquisition["test_timeseries"] + _compare_attrs(acq, py_acq) + # data and timeseries should be equal + assert np.array_equal(acq.data[:], py_acq.data[:]) + assert np.array_equal(acq.timestamps[:], py_acq.timestamps[:]) + + +def test_position(read_nwbfile, read_pynwb): + trials = read_nwbfile.intervals.trials[:] + py_trials = read_pynwb.trials.to_dataframe() + pd.testing.assert_frame_equal(py_trials, trials) + + spatial = read_nwbfile.processing["behavior"].Position.SpatialSeries + py_spatial = read_pynwb.processing["behavior"]["Position"]["SpatialSeries"] + _compare_attrs(spatial, py_spatial) + assert np.array_equal(spatial[:], py_spatial.data[:]) + assert np.array_equal(spatial.timestamps[:], py_spatial.timestamps[:]) + + +def test_ecephys(read_nwbfile, read_pynwb): + pass + + +def test_units(read_nwbfile, read_pynwb): + pass + + +def test_icephys(read_nwbfile, read_pynwb): + pass + + +def test_ca_imaging(read_nwbfile, read_pynwb): + pass + + +def test_read_from_yaml(nwb_file): + """ + Read data from a yaml-fied NWB file + """ + pass diff --git a/nwb_models/README.md b/nwb_models/README.md index cc36d0ab..47582023 100644 --- a/nwb_models/README.md +++ b/nwb_models/README.md @@ -1 +1,3 @@ # nwb-models + +(README forthcoming, for now see [`nwb-linkml`](https://pypi.org/project/nwb-linkml)) \ No newline at end of file diff --git a/nwb_models/pyproject.toml b/nwb_models/pyproject.toml index 21078a25..59b0b6d8 100644 --- a/nwb_models/pyproject.toml +++ b/nwb_models/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nwb-models" -version = "0.1.0" +version = "0.2.0" description = "Pydantic/LinkML models for Neurodata Without Borders" authors = [ {name = "sneakers-the-rat", email = "sneakers-the-rat@protonmail.com"}, diff --git a/nwb_models/src/nwb_models/models/pydantic/__init__.py b/nwb_models/src/nwb_models/models/pydantic/__init__.py index fa3cf1e3..e69de29b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/__init__.py +++ b/nwb_models/src/nwb_models/models/pydantic/__init__.py @@ -1 +0,0 @@ -from .pydantic.core.v2_7_0.namespace import * diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py index db6e75cd..263d3893 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py index 31bf3222..5691daba 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py index bf153874..ab24817c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py index 7a99a152..136ec40a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py index 7475c410..4ab3c01a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_2_0.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py index e03f10b3..ae163918 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py @@ -24,7 +24,12 @@ from ...core.v2_2_0.core_nwb_misc import Units from ...core.v2_2_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -35,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +59,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -464,7 +500,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -473,7 +509,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -482,7 +518,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -491,7 +527,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -505,14 +541,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py index be3a0abf..439d5af5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py @@ -26,7 +26,12 @@ TimeSeriesSync, ) from ...core.v2_2_0.core_nwb_device import Device -from ...hdmf_common.v1_1_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py index 73b924f6..33784d6c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py index f83925fc..e8a4896b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_1_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py index 853250aa..998dda00 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py index 6ce44c7a..70db9d78 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py index ab2f91eb..17edeecd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py @@ -31,7 +31,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +50,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -166,17 +197,16 @@ class RetinotopyImage(GrayscaleImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImagingRetinotopy(NWBDataInterface): @@ -204,7 +234,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_1_power_map: Named[Optional[AxisMap]] = Field( + axis_1_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response on the first measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.""", json_schema_extra={ @@ -228,7 +258,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_2_power_map: Named[Optional[AxisMap]] = Field( + axis_2_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response to stimulus on the second measured axis.""", json_schema_extra={ @@ -306,17 +336,16 @@ class ImagingRetinotopyFocalDepthImage(RetinotopyImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py index 710c80c4..d4b265da 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py @@ -149,7 +149,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -168,6 +168,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py index 635f77af..f0f43be6 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py index c0a675be..e96918cb 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py index 1da62bba..80de9c0a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py index 25525e86..169dd5e4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py index 247b6679..ed1353e2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_2_1.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_2.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_2.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py index 49ebbf05..b5a0b9b8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py @@ -24,7 +24,12 @@ from ...core.v2_2_1.core_nwb_misc import Units from ...core.v2_2_1.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_1.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_2.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_2.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -35,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +59,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -464,7 +500,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -473,7 +509,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -482,7 +518,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -491,7 +527,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -505,14 +541,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py index 34a9d423..991c1e8c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py @@ -26,7 +26,12 @@ TimeSeriesSync, ) from ...core.v2_2_1.core_nwb_device import Device -from ...hdmf_common.v1_1_2.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_2.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py index 08247784..52c10a5c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py index ecd09460..19a036f3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_1_2.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py index 1577358c..609baf06 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py index ca843fc0..a951c51d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py index 69e47a34..1c6f4ad2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py @@ -31,7 +31,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +50,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -166,17 +197,16 @@ class RetinotopyImage(GrayscaleImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImagingRetinotopy(NWBDataInterface): @@ -204,7 +234,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_1_power_map: Named[Optional[AxisMap]] = Field( + axis_1_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response on the first measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.""", json_schema_extra={ @@ -228,7 +258,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_2_power_map: Named[Optional[AxisMap]] = Field( + axis_2_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response to stimulus on the second measured axis.""", json_schema_extra={ @@ -306,17 +336,16 @@ class ImagingRetinotopyFocalDepthImage(RetinotopyImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py index 5dd8a411..7f2ade1c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py @@ -149,7 +149,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -168,6 +168,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py index dd580be1..956e37d0 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py index bb900e7c..271fceb4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py index dbb96bf2..28aa9543 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py index 5749b004..9664726a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py index 6e766ad4..c12a9659 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_2_2.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py index 3824ab21..ec664713 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py @@ -24,7 +24,12 @@ from ...core.v2_2_2.core_nwb_misc import Units from ...core.v2_2_2.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_2.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -35,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +59,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -464,7 +500,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -473,7 +509,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -482,7 +518,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -491,7 +527,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -505,14 +541,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py index 9d4a6964..9b7729d7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py @@ -26,7 +26,12 @@ TimeSeriesSync, ) from ...core.v2_2_2.core_nwb_device import Device -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py index fec53fa2..6e805b1b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py index 5c32d21d..d80af522 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py index 8c1e21d4..debdaf9e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py index b7ed446a..e7b56da4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py index f92004dd..bfa2ad5e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py index 536af15d..9ba793bc 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py @@ -152,7 +152,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -171,6 +171,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py index 82fa6fff..0e814861 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py index e14c8d59..42613b42 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py index 09cf60b2..1aeeb6c7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py index 25a0a426..d4f5172d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py index 3cd50787..61f894bb 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_2_4.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py index 05420716..9167a4da 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py @@ -25,7 +25,12 @@ from ...core.v2_2_4.core_nwb_misc import Units from ...core.v2_2_4.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_4.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -36,7 +41,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +60,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -440,7 +476,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -449,7 +485,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -458,7 +494,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -467,7 +503,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -481,14 +517,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py index 8f20762c..8067eb7c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py @@ -26,7 +26,12 @@ TimeSeriesSync, ) from ...core.v2_2_4.core_nwb_device import Device -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py index 850e89ff..05c1d6e6 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py index 22fc753f..5ff807ca 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py index 50d25ad4..20f63530 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py index 1a0bf166..b91e4482 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -322,7 +354,7 @@ class PlaneSegmentation(DynamicTable): None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -338,7 +370,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -373,14 +405,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py index 111a502c..362bc592 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py index 10795d3b..23ec3dd7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py @@ -159,7 +159,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -178,6 +178,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py index 5012a43b..86fe03fd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py index e7c936a3..f4f5e960 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py index 5d308f9e..5abfc5d7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py index 4e993f0d..48d25034 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py index bd7e37eb..6a8ba5a3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_2_5.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py index 1998de24..59aa79e0 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py @@ -25,7 +25,12 @@ from ...core.v2_2_5.core_nwb_misc import Units from ...core.v2_2_5.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_5.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -36,7 +41,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +60,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -440,7 +476,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -449,7 +485,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -458,7 +494,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -467,7 +503,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -481,14 +517,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py index 5a576638..ee68bffd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py @@ -26,7 +26,12 @@ TimeSeriesSync, ) from ...core.v2_2_5.core_nwb_device import Device -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py index 7f20c3a7..f3d0d5f8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py index 36c0f1e2..5faeb055 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py index 294866bf..6c81182f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py index 4860989b..98c3a537 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -324,7 +356,7 @@ class PlaneSegmentation(DynamicTable): None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -340,7 +372,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -375,14 +407,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py index 6f058143..5466646b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py index 5f823795..5d12f36d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py @@ -159,7 +159,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -178,6 +178,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py index 34bb4c68..ad3c5f4c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py @@ -23,7 +23,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -42,6 +42,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -274,7 +305,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -294,7 +325,7 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") + image: List[str] = Field(..., description="""Images stored in this collection.""") # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py index ef008270..8358db6c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -167,7 +198,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -182,7 +213,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -197,7 +228,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -212,7 +243,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -227,7 +258,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -242,7 +273,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -257,7 +288,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py index 26e327b6..5c0f4513 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py index 7092c138..2676bd5f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py index d5015fe1..93ea1ba5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_3_0.core_nwb_base import TimeSeries -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class TimeIntervalsTimeseries(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py index 2000dfde..d6920651 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py @@ -25,7 +25,7 @@ from ...core.v2_3_0.core_nwb_misc import Units from ...core.v2_3_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_3_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -152,28 +183,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -206,12 +237,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[TimeSeries]] = Field( + templates: Optional[Dict[str, TimeSeries]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, @@ -289,11 +320,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -308,12 +339,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -353,7 +384,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -428,8 +459,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -440,7 +477,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -449,7 +486,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -458,7 +495,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -467,7 +504,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -481,14 +518,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -511,7 +545,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -542,7 +576,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py index aad36313..1fb2a046 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py @@ -26,7 +26,12 @@ TimeSeriesSync, ) from ...core.v2_3_0.core_nwb_device import Device -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -879,8 +915,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -899,14 +941,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py index fb2bc82e..8758ca8f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py @@ -23,7 +23,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -42,6 +42,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -85,17 +116,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -108,17 +138,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -131,17 +168,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py index 8d984db7..ac3b3661 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py index 2e49cf9d..bf95c5ca 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py index 29cac39f..670269a7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -283,7 +315,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -298,7 +330,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -313,7 +345,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -329,11 +361,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -349,7 +388,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -365,7 +404,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -384,38 +423,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -538,7 +550,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -652,7 +664,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -694,7 +706,6 @@ class CorrectedImageStack(NWBDataInterface): Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py index d58860e1..5c786580 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py index b5ffa4b6..2125d572 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py @@ -117,7 +117,6 @@ MotionCorrection, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -134,7 +133,7 @@ ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -162,7 +161,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -181,6 +180,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py index b557e8c8..f8b6d994 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -88,7 +119,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -469,7 +500,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -489,7 +520,7 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") + image: List[str] = Field(..., description="""Images stored in this collection.""") # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py index c96aee79..7c0abb87 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -167,7 +198,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -182,7 +213,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -197,7 +228,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -212,7 +243,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -227,7 +258,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -242,7 +273,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -257,7 +288,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py index ebf61b7c..436d2d41 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py index bfe65b35..ac26b290 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py index 603c5d3b..25894a36 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_4_0.core_nwb_base import TimeSeries -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class TimeIntervalsTimeseries(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py index 186458d9..84d5b9a3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py @@ -33,7 +33,7 @@ from ...core.v2_4_0.core_nwb_misc import Units from ...core.v2_4_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_4_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -160,28 +191,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -214,12 +245,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[TimeSeries]] = Field( + templates: Optional[Dict[str, TimeSeries]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, @@ -297,11 +328,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -316,12 +347,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -361,7 +392,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -436,8 +467,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -448,7 +485,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -457,7 +494,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -466,7 +503,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -475,7 +512,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -489,14 +526,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -519,7 +553,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -571,7 +605,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py index aa2acccb..d4ebcb34 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py @@ -31,6 +31,7 @@ AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -886,8 +918,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -906,14 +944,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -936,21 +971,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -989,14 +1027,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1035,14 +1070,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1094,21 +1126,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1150,14 +1179,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1238,14 +1264,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1317,14 +1340,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1398,14 +1418,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py index 5d1e5d8f..8fd3288c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py @@ -23,7 +23,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -42,6 +42,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -85,17 +116,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -108,17 +138,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -131,17 +168,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py index 6c06a177..3ab6b758 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py index 07100b30..350a3984 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py index 0d335ca7..9f6e191a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -283,7 +315,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -298,7 +330,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -313,7 +345,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -329,11 +361,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -349,7 +388,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -365,7 +404,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -384,38 +423,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -538,7 +550,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -652,7 +664,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -694,7 +706,6 @@ class CorrectedImageStack(NWBDataInterface): Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py index a42a469f..ffc194ec 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py index 4bc04cdd..620dcf2f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py @@ -130,7 +130,6 @@ MotionCorrection, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -147,7 +146,7 @@ ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -175,7 +174,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -194,6 +193,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py index 3a2170a1..2db9763f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py @@ -47,7 +47,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -66,6 +66,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -99,7 +130,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -520,7 +551,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -540,8 +571,8 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") - order_of_images: Named[Optional[ImageReferences]] = Field( + image: List[str] = Field(..., description="""Images stored in this collection.""") + order_of_images: Optional[Named[ImageReferences]] = Field( None, description="""Ordered dataset of references to Image objects stored in the parent group. Each Image object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.""", json_schema_extra={ diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py index d4e6a037..89c1038a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -169,7 +200,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -184,7 +215,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -199,7 +230,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -214,7 +245,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -229,7 +260,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -244,7 +275,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -259,7 +290,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py index 16f07fcf..e4cf279b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py index 343564a7..91c22222 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py index 2fafb906..ab92eb72 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_5_0.core_nwb_base import TimeSeriesReferenceVectorData -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -148,7 +184,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + timeseries: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""An index into a TimeSeries object.""", json_schema_extra={ @@ -160,7 +196,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -177,14 +213,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py index 039eb68c..6c056a6e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py @@ -34,7 +34,7 @@ from ...core.v2_5_0.core_nwb_misc import Units from ...core.v2_5_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_5_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -45,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -64,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -161,28 +192,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -215,12 +246,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[Union[Images, TimeSeries]]] = Field( + templates: Optional[Dict[str, Union[Images, TimeSeries]]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={ @@ -300,11 +331,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -319,12 +350,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -364,7 +395,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -385,7 +416,7 @@ class ExtracellularEphysElectrodes(DynamicTable): "linkml_meta": {"equals_string": "electrodes", "ifabsent": "string(electrodes)"} }, ) - x: VectorData[Optional[NDArray[Any, float]]] = Field( + x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate of the channel location in the brain (+x is posterior).""", json_schema_extra={ @@ -394,7 +425,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - y: VectorData[Optional[NDArray[Any, float]]] = Field( + y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate of the channel location in the brain (+y is inferior).""", json_schema_extra={ @@ -403,7 +434,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - z: VectorData[Optional[NDArray[Any, float]]] = Field( + z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate of the channel location in the brain (+z is right).""", json_schema_extra={ @@ -412,7 +443,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - imp: VectorData[Optional[NDArray[Any, float]]] = Field( + imp: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""Impedance of the channel, in ohms.""", json_schema_extra={ @@ -430,7 +461,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - filtering: VectorData[Optional[NDArray[Any, str]]] = Field( + filtering: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of hardware filtering, including the filter name and frequency cutoffs.""", json_schema_extra={ @@ -439,8 +470,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -451,7 +488,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -460,7 +497,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -469,7 +506,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -478,7 +515,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".""", json_schema_extra={ @@ -492,14 +529,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -522,7 +556,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -574,7 +608,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py index bef122ce..b500a82d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py @@ -31,6 +31,7 @@ AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -887,8 +919,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -907,14 +945,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -937,21 +972,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -990,14 +1028,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1036,14 +1071,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1095,21 +1127,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1151,14 +1180,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1239,14 +1265,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1318,14 +1341,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1399,14 +1419,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py index 21e5d0a8..520d2495 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -91,17 +122,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -114,17 +144,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -137,17 +174,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py index 6be8f5c2..7901288b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py index 5d3e9ffb..e39e6b29 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py index fef5a92b..6107f4ca 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -283,7 +315,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -298,7 +330,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -313,7 +345,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -329,11 +361,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -349,7 +388,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -365,7 +404,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -384,38 +423,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -538,7 +550,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -652,7 +664,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -694,7 +706,6 @@ class CorrectedImageStack(NWBDataInterface): Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py index e3994489..b72f7b44 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py index 5692d119..7aaa8a2c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py @@ -131,7 +131,6 @@ MotionCorrection, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -148,7 +147,7 @@ ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -176,7 +175,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -195,6 +194,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py index c4d356f0..4837ae7f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py @@ -47,7 +47,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -66,6 +66,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -99,7 +130,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -520,7 +551,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -540,8 +571,8 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") - order_of_images: Named[Optional[ImageReferences]] = Field( + image: List[str] = Field(..., description="""Images stored in this collection.""") + order_of_images: Optional[Named[ImageReferences]] = Field( None, description="""Ordered dataset of references to Image objects stored in the parent group. Each Image object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.""", json_schema_extra={ diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py index 07f51659..7e4ad59e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -169,7 +200,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -184,7 +215,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -199,7 +230,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -214,7 +245,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -229,7 +260,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -244,7 +275,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -259,7 +290,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py index c1a89c40..c57186b1 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py index d83d6501..05290353 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py index 46da3616..d3fa53b4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_6_0_alpha.core_nwb_base import TimeSeriesReferenceVectorData -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -148,7 +184,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + timeseries: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""An index into a TimeSeries object.""", json_schema_extra={ @@ -160,7 +196,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -177,14 +213,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py index 6c39f8e9..975e51c4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py @@ -34,7 +34,7 @@ from ...core.v2_6_0_alpha.core_nwb_misc import Units from ...core.v2_6_0_alpha.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_6_0_alpha.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -45,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -64,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -161,28 +192,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -215,12 +246,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[Union[Images, TimeSeries]]] = Field( + templates: Optional[Dict[str, Union[Images, TimeSeries]]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={ @@ -300,11 +331,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -319,12 +350,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -364,7 +395,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -385,7 +416,7 @@ class ExtracellularEphysElectrodes(DynamicTable): "linkml_meta": {"equals_string": "electrodes", "ifabsent": "string(electrodes)"} }, ) - x: VectorData[Optional[NDArray[Any, float]]] = Field( + x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate of the channel location in the brain (+x is posterior).""", json_schema_extra={ @@ -394,7 +425,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - y: VectorData[Optional[NDArray[Any, float]]] = Field( + y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate of the channel location in the brain (+y is inferior).""", json_schema_extra={ @@ -403,7 +434,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - z: VectorData[Optional[NDArray[Any, float]]] = Field( + z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate of the channel location in the brain (+z is right).""", json_schema_extra={ @@ -412,7 +443,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - imp: VectorData[Optional[NDArray[Any, float]]] = Field( + imp: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""Impedance of the channel, in ohms.""", json_schema_extra={ @@ -430,7 +461,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - filtering: VectorData[Optional[NDArray[Any, str]]] = Field( + filtering: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of hardware filtering, including the filter name and frequency cutoffs.""", json_schema_extra={ @@ -439,8 +470,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -451,7 +488,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -460,7 +497,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -469,7 +506,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -478,7 +515,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".""", json_schema_extra={ @@ -492,14 +529,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -522,7 +556,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -574,7 +608,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py index 8142b53e..1f9c04ba 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py @@ -31,6 +31,7 @@ AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -887,8 +919,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -907,14 +945,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -937,21 +972,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -990,14 +1028,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1036,14 +1071,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1095,21 +1127,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1151,14 +1180,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1239,14 +1265,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1318,14 +1341,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1399,14 +1419,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py index e0506e95..af69abe5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -91,17 +122,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -114,17 +144,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -137,17 +174,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py index ee349a1e..5c287364 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit in seconds.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py index 5565ce8a..42fe82fc 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py index 76d0e678..f6acd6c7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -382,7 +414,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -397,7 +429,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -412,7 +444,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -428,11 +460,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -448,7 +487,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -464,7 +503,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -483,38 +522,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -637,7 +649,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -751,7 +763,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -794,7 +806,6 @@ class CorrectedImageStack(NWBDataInterface): Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py index b3017f18..3a085f7f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py index a6c0e87c..21b70464 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py @@ -133,7 +133,6 @@ OnePhotonSeries, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -150,7 +149,7 @@ ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -178,7 +177,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -197,6 +196,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py index 961bfd4f..a645a2f7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py @@ -47,7 +47,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -66,6 +66,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -99,7 +130,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -520,7 +551,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -540,8 +571,8 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") - order_of_images: Named[Optional[ImageReferences]] = Field( + image: List[str] = Field(..., description="""Images stored in this collection.""") + order_of_images: Optional[Named[ImageReferences]] = Field( None, description="""Ordered dataset of references to Image objects stored in the parent group. Each Image object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.""", json_schema_extra={ diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py index 43c19369..836c2e2a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -169,7 +200,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -184,7 +215,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -199,7 +230,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -214,7 +245,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -229,7 +260,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -244,7 +275,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -259,7 +290,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py index 38677447..59b53c8e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py index 89eeeb0b..dc96a983 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py index 95178b04..e8b5539f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ ) from ...core.v2_7_0.core_nwb_base import TimeSeriesReferenceVectorData -from ...hdmf_common.v1_8_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_8_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -148,7 +184,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + timeseries: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""An index into a TimeSeries object.""", json_schema_extra={ @@ -160,7 +196,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -177,14 +213,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py index fe7fabd9..038a4ae6 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py @@ -34,7 +34,7 @@ from ...core.v2_7_0.core_nwb_misc import Units from ...core.v2_7_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_7_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_8_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_8_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -45,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -64,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -161,28 +192,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -215,7 +246,7 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[Union[DynamicTable, NWBDataInterface, TimeSeries]]] = Field( + presentation: Optional[Dict[str, Union[DynamicTable, NWBDataInterface, TimeSeries]]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={ @@ -228,7 +259,7 @@ class NWBFileStimulus(ConfiguredBaseModel): } }, ) - templates: Optional[List[Union[Images, TimeSeries]]] = Field( + templates: Optional[Dict[str, Union[Images, TimeSeries]]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={ @@ -308,11 +339,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -327,12 +358,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -372,7 +403,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -393,7 +424,7 @@ class ExtracellularEphysElectrodes(DynamicTable): "linkml_meta": {"equals_string": "electrodes", "ifabsent": "string(electrodes)"} }, ) - x: VectorData[Optional[NDArray[Any, float]]] = Field( + x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate of the channel location in the brain (+x is posterior).""", json_schema_extra={ @@ -402,7 +433,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - y: VectorData[Optional[NDArray[Any, float]]] = Field( + y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate of the channel location in the brain (+y is inferior).""", json_schema_extra={ @@ -411,7 +442,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - z: VectorData[Optional[NDArray[Any, float]]] = Field( + z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate of the channel location in the brain (+z is right).""", json_schema_extra={ @@ -420,7 +451,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - imp: VectorData[Optional[NDArray[Any, float]]] = Field( + imp: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""Impedance of the channel, in ohms.""", json_schema_extra={ @@ -438,7 +469,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - filtering: VectorData[Optional[NDArray[Any, str]]] = Field( + filtering: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of hardware filtering, including the filter name and frequency cutoffs.""", json_schema_extra={ @@ -447,8 +478,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -459,7 +496,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -468,7 +505,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -477,7 +514,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -486,7 +523,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".""", json_schema_extra={ @@ -500,14 +537,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -530,7 +564,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -582,7 +616,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py index 9e728129..c1818b4b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py @@ -31,6 +31,7 @@ AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -887,8 +919,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -907,14 +945,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -937,21 +972,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -986,7 +1024,7 @@ class IntracellularStimuliTable(DynamicTable): } }, ) - stimulus_template: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + stimulus_template: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""Column storing the reference to the stimulus template for the recording (rows).""", json_schema_extra={ @@ -1002,14 +1040,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1048,14 +1083,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1107,21 +1139,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1163,14 +1192,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1251,14 +1277,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1330,14 +1353,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1411,14 +1431,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py index d98ffe11..6e971729 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -91,17 +122,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -114,17 +144,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -137,17 +174,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py index 31c081bc..1eb2c3a0 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...hdmf_common.v1_8_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit in seconds.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py index 5eb87cf0..626a28c5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py index 3f8d8ebe..d4620642 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_8_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -382,7 +414,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -397,7 +429,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -412,7 +444,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -428,11 +460,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -448,7 +487,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -464,7 +503,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -483,38 +522,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -637,7 +649,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -751,7 +763,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -794,7 +806,6 @@ class CorrectedImageStack(NWBDataInterface): Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py index 909aaf33..26f2f924 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py index 9256d2fb..5747cdef 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py @@ -133,7 +133,6 @@ OnePhotonSeries, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -150,7 +149,7 @@ ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_8_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_8_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -179,7 +178,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -198,6 +197,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py index c71894ee..56af1b8f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py index fb5ae6fb..e52b2945 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -96,7 +127,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -298,8 +329,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -383,7 +417,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -491,11 +525,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -504,6 +541,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -522,17 +560,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -565,9 +611,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -617,10 +663,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -650,28 +699,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -679,8 +729,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -738,14 +787,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -779,12 +833,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -826,7 +887,7 @@ class Index(Data): ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex(0)+1]. The second vector is at VectorData[VectorIndex(0)+1:VectorIndex(1)+1], and so on. """ @@ -839,7 +900,7 @@ class VectorData(VectorDataMixin): description: str = Field(..., description="""Description of what these vectors represent.""") -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. """ @@ -854,7 +915,7 @@ class VectorIndex(VectorIndexMixin): ) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -866,9 +927,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -898,7 +963,7 @@ class Container(ConfiguredBaseModel): name: str = Field(...) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). Apart from a column that contains unique identifiers for each row there are no other required datasets. Users are free to add any number of VectorData objects here. Table functionality is already supported through compound types, which is analogous to storing an array-of-structs. DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. For example, DynamicTable was originally developed for storing trial data and spike unit metadata. Both of these use cases are expected to produce relatively small tables, so the spatial locality of multiple datasets present in a DynamicTable is not expected to have a significant performance impact. Additionally, requirements of trial and unit metadata tables are sufficiently diverse that performance implications can be overlooked in favor of usability. """ @@ -913,14 +978,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py index 1a6e22f0..dcc5707b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py index bd668328..8ce7f438 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py index 50f164b6..9065b81b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -96,7 +127,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -298,8 +329,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -383,7 +417,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -491,11 +525,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -504,6 +541,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -522,17 +560,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -565,9 +611,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -617,10 +663,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -650,28 +699,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -679,8 +729,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -738,14 +787,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -779,12 +833,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -826,7 +887,7 @@ class Index(Data): ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex(0)+1]. The second vector is at VectorData[VectorIndex(0)+1:VectorIndex(1)+1], and so on. """ @@ -839,7 +900,7 @@ class VectorData(VectorDataMixin): description: str = Field(..., description="""Description of what these vectors represent.""") -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. """ @@ -854,7 +915,7 @@ class VectorIndex(VectorIndexMixin): ) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -866,9 +927,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -898,7 +963,7 @@ class Container(ConfiguredBaseModel): name: str = Field(...) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). Apart from a column that contains unique identifiers for each row there are no other required datasets. Users are free to add any number of VectorData objects here. Table functionality is already supported through compound types, which is analogous to storing an array-of-structs. DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. For example, DynamicTable was originally developed for storing trial data and spike unit metadata. Both of these use cases are expected to produce relatively small tables, so the spatial locality of multiple datasets present in a DynamicTable is not expected to have a significant performance impact. Additionally, requirements of trial and unit metadata tables are sufficiently diverse that performance implications can be overlooked in favor of usability. """ @@ -913,14 +978,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py index 786e141a..0f669854 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py index 09ea0f19..c0f7fcc1 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py index 0d16b558..749fab91 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -96,7 +127,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -298,8 +329,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -383,7 +417,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -491,11 +525,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -504,6 +541,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -522,17 +560,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -565,9 +611,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -617,10 +663,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -650,28 +699,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -679,8 +729,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -738,14 +787,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -779,12 +833,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -826,7 +887,7 @@ class Index(Data): ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex(0)+1]. The second vector is at VectorData[VectorIndex(0)+1:VectorIndex(1)+1], and so on. """ @@ -837,17 +898,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. """ @@ -865,7 +919,7 @@ class VectorIndex(VectorIndexMixin): ) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -877,9 +931,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -917,7 +975,7 @@ class Container(ConfiguredBaseModel): name: str = Field(...) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). Apart from a column that contains unique identifiers for each row there are no other required datasets. Users are free to add any number of VectorData objects here. Table functionality is already supported through compound types, which is analogous to storing an array-of-structs. DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. For example, DynamicTable was originally developed for storing trial data and spike unit metadata. Both of these use cases are expected to produce relatively small tables, so the spatial locality of multiple datasets present in a DynamicTable is not expected to have a significant performance impact. Additionally, requirements of trial and unit metadata tables are sufficiently diverse that performance implications can be overlooked in favor of usability. """ @@ -932,14 +990,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py index 1458d9b4..c505d772 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py index aa2b460a..656629d7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py index a0a70de5..13824fed 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py index 647bc234..fdd6bcc4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -912,7 +970,7 @@ class VocabData(VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -927,14 +985,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py index f56d638f..25e56518 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py @@ -35,7 +35,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +54,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py index 7476a40a..affaa593 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py index c2222d10..01484f33 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py index d1c0692f..cc9029d9 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -912,7 +970,7 @@ class VocabData(VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -927,14 +985,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py index f8a0d6f2..1338679b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py @@ -35,7 +35,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +54,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py index d4846522..a7ed66d3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py index 8f8f7118..4d4850c9 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py index 4d93c709..2620eb6f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py index ece0532f..a55c2129 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -912,7 +970,7 @@ class VocabData(VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -927,14 +985,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py index dcb742c9..040adf72 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py @@ -37,7 +37,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +56,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py index f47e8ca6..02d67bfd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py index af8cc73e..ad70998e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py index d1202efa..a730ec11 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py index b110f2db..0a85a76b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_4_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_4_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py index 2412f823..7c62f934 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py index 21258d8c..d434cd91 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py index c1b61d90..27a287c8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py index d5c14d92..6e04fd09 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py index f3c24b4e..df229481 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py index fce455bb..4e921ccd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py index 4a91362f..3112a4fa 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py index 442efe7c..fa4ea723 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_5_1.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_1.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py index d53c4b5e..57c9079d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py index d60f430f..73a6043a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py index da5a37fe..0759b510 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py index f9dd0a82..981e600e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_6_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_6_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py index 7872584d..e785e04d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py index e21dca7d..b2fe190a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py index 9638faab..e805fe75 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py index a3f0b147..4aaa46dd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_7_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_7_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py index 8b4d98fb..7731368c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py index 4da904e9..7a3e72cd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py index b62156a9..8f0d610e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ def __getitem__( # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ def create_colnames(cls, model: Dict[str, Any]) -> Dict: and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ def cast_extra_columns(cls, model: Dict[str, Any]) -> Dict: if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ def __getitem__( elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ def __getitem__( "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ def create_categories(cls, model: Dict[str, Any]) -> Dict: model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ def ensure_equal_length_cols(self) -> "DynamicTableMixin": ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py index 78035bec..dd09b7f2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_8_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_8_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py index c5848aae..ad617da3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py index 5d0a97e7..cda720e1 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py index 3389dc8f..3429a1e7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_4_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_4_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py index a840dc1c..1a88edc1 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py index 86907725..8d5af36b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py index 8b52b450..c697f831 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_5_1.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_1.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py index 4fdfb371..cbd0ad9f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py index 1324db15..9f337fae 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py index c4b9cd86..e1a12cab 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_6_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_6_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py index 43eab2a1..0551cfd5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py index 172b75b8..09e6f052 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py index 14fcb967..c904202b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_7_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_7_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -40,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -59,6 +59,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py index a18965fc..714ae528 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py index 712c32eb..d3132cdc 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py index b4c0f3a3..281e5b2f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_8_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_8_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -40,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -59,6 +59,37 @@ def __getitem__(self, val: Union[int, slice]) -> Any: else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml index 69204844..9019ee0e 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml index 7f406bcd..785ebbb6 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml index 1e11ca41..9b619e37 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml index 4beec017..1db1fbac 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml index 4218d3b3..3a7025a3 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml index 24a4bbf9..b11b02c2 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -33,6 +34,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -74,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -85,6 +88,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -95,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -120,6 +125,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -128,6 +134,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -171,6 +178,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -181,6 +190,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -258,6 +268,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -289,6 +300,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -313,6 +325,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml index 1b742038..f63c218c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml index 7881fcf3..8d60d56a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml index 2f3dd970..7a934612 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml index ce14120d..4eb778da 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,12 +65,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries description: An index into a TimeSeries object. range: TimeIntervals__timeseries required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -83,6 +86,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true TimeIntervals__timeseries: name: TimeIntervals__timeseries @@ -92,6 +96,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true equals_string: timeseries @@ -122,3 +127,4 @@ classes: range: TimeSeries required: false multivalued: false + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml index 0b76f4f9..a3eb4631 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -280,6 +291,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -375,6 +387,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -402,6 +415,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -418,18 +433,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -454,6 +475,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -473,6 +495,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -482,12 +505,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -496,6 +523,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -559,9 +587,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -614,6 +646,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -631,12 +664,16 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: The table which groups different PatchClampSeries together. range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -646,6 +683,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -656,18 +694,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -675,6 +719,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -682,6 +728,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -692,6 +739,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml index d93bb528..26823bef 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -152,6 +159,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -188,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -196,6 +205,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -204,6 +214,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -229,6 +240,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -237,48 +249,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -287,6 +307,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -310,6 +331,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -331,6 +353,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -352,6 +375,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -374,6 +398,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -396,6 +421,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -418,6 +444,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -440,6 +467,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -462,6 +490,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -470,6 +499,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -478,6 +508,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -501,6 +532,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -555,6 +587,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -566,6 +599,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -580,9 +614,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -596,4 +634,5 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml index bbbcfced..adfab1b5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -95,6 +124,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -112,6 +142,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -126,6 +157,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -163,6 +195,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -173,6 +206,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -188,6 +222,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -250,6 +285,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -269,6 +305,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml index 89d5ee00..c2323b8a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml index 3148b988..0dc7be01 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml index e9c680d0..40860fc5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -60,6 +61,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -72,6 +74,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -102,6 +105,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -156,15 +160,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -178,6 +195,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -186,6 +204,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -199,6 +218,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -207,6 +227,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -223,22 +244,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -249,6 +259,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -286,6 +297,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -328,6 +340,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -371,6 +384,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -379,6 +393,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -387,6 +402,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -415,6 +431,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -423,6 +441,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -436,6 +455,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -485,6 +505,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -515,6 +536,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -543,6 +565,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -579,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -587,6 +611,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -594,6 +620,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -602,6 +630,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml index cc06e90b..97007ea0 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml index 83697071..1bfb9119 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -114,6 +118,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -125,6 +130,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -135,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -160,6 +167,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -168,6 +176,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -211,6 +220,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -221,6 +232,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -298,6 +310,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -329,6 +342,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -353,6 +367,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml index ba292369..47aa7523 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml index fc320af6..307f8469 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml index f0eccd67..4d8e5393 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml index c3fb2cb3..e264a547 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,12 +65,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries description: An index into a TimeSeries object. range: TimeIntervals__timeseries required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -83,6 +86,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true TimeIntervals__timeseries: name: TimeIntervals__timeseries @@ -92,6 +96,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true equals_string: timeseries @@ -122,3 +127,4 @@ classes: range: TimeSeries required: false multivalued: false + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml index 13bf8a1e..f81b1576 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -280,6 +291,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -375,6 +387,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -402,6 +415,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -418,18 +433,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -454,6 +475,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -473,6 +495,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -482,12 +505,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -496,6 +523,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -559,9 +587,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -614,6 +646,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -632,6 +665,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -641,6 +676,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -658,6 +695,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -666,6 +705,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -675,6 +716,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -684,6 +727,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -691,6 +736,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -700,6 +747,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -710,18 +758,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -729,6 +783,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -736,6 +792,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -746,6 +803,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml index 346751e9..d3a808f4 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -152,6 +159,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -188,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -196,6 +205,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -204,6 +214,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -229,6 +240,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -237,48 +249,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -287,6 +307,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -310,6 +331,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -331,6 +353,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -352,6 +375,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -374,6 +398,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -396,6 +421,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -418,6 +444,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -440,6 +467,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -462,6 +490,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -470,6 +499,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -478,6 +508,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -501,6 +532,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -555,6 +587,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -569,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -583,9 +617,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -599,6 +637,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -607,6 +646,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -619,9 +659,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -630,6 +674,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -653,6 +698,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -661,6 +707,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -684,6 +731,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -703,6 +751,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -724,18 +773,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -747,6 +802,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -757,6 +813,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -770,6 +827,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -780,6 +838,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -790,6 +849,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -801,6 +861,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -811,6 +872,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -824,6 +886,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -843,6 +906,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -853,6 +917,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -864,6 +929,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -874,6 +940,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -887,6 +954,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -897,6 +965,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -907,6 +976,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -916,6 +986,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -925,6 +996,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -938,6 +1010,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -947,6 +1020,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -957,3 +1031,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml index ac28a308..fec75ec8 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -251,6 +286,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -270,6 +306,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml index 97927d61..ec02fc46 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml index 1add7786..cbe1a6de 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml index c6215f13..aec85470 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -60,6 +61,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -72,6 +74,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -102,6 +105,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -156,15 +160,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -178,6 +195,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -186,6 +204,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -199,6 +218,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -207,6 +227,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -223,22 +244,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -249,6 +259,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -286,6 +297,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -328,6 +340,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -371,6 +384,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -379,6 +393,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -387,6 +402,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -415,6 +431,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -423,6 +441,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -436,6 +455,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -485,6 +505,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -515,6 +536,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -543,6 +565,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -579,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -587,6 +611,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -594,6 +620,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -602,6 +630,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml index f433f10a..f30f06fb 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml index 373ff4d8..547dd4c1 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -113,6 +117,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true value: @@ -125,6 +130,8 @@ classes: range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true NWBContainer: name: NWBContainer @@ -134,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -145,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -155,6 +164,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -180,6 +190,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -188,6 +199,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -231,6 +243,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -241,6 +255,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -327,6 +342,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -358,6 +374,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -384,6 +401,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: @@ -413,4 +431,5 @@ classes: range: ImageReferences required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml index fd2c46fa..94ff5f81 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml index 3f1acc9c..d2ec1a5a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml index ce256eb8..b611d74c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml index 3764b003..9857394b 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,6 +65,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries annotations: @@ -77,6 +79,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -90,4 +93,5 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml index f4680499..01ef5b58 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -281,6 +292,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -376,6 +388,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -403,6 +416,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -419,18 +434,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -455,6 +476,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -474,6 +496,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -483,12 +506,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -497,6 +524,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -560,9 +588,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -617,6 +649,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -635,6 +668,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -644,6 +679,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -661,6 +698,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -669,6 +708,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -678,6 +719,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -687,6 +730,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -694,6 +739,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -703,6 +750,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -713,18 +761,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -732,6 +786,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -739,6 +795,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -749,6 +806,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml index bdd9dd51..257b07b5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -153,6 +160,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -189,6 +197,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -197,6 +206,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -205,6 +215,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -231,6 +242,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -239,48 +251,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -289,6 +309,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -313,6 +334,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -334,6 +356,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -355,6 +378,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -377,6 +401,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -399,6 +424,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -421,6 +447,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -443,6 +470,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -465,6 +493,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -473,6 +502,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -481,6 +511,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -505,6 +536,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true cell_id: @@ -565,6 +597,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -579,6 +612,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -593,9 +627,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -609,6 +647,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -617,6 +656,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -629,9 +669,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -640,6 +684,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -663,6 +708,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -671,6 +717,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -694,6 +741,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -713,6 +761,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -734,18 +783,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -757,6 +812,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -767,6 +823,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -780,6 +837,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -790,6 +848,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -800,6 +859,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -811,6 +871,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -821,6 +882,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -834,6 +896,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -853,6 +916,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -863,6 +927,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -874,6 +939,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -884,6 +950,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -897,6 +964,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -907,6 +975,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -917,6 +986,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -926,6 +996,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -935,6 +1006,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -948,6 +1020,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -957,6 +1030,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -967,3 +1041,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml index 0f6efd9a..dd4d2f47 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -252,6 +287,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -272,6 +308,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -283,6 +320,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Images - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml index f6639946..5bfeb440 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml index adadc3e1..8c6b0763 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml index 9cd8b1ea..17bb442c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -60,6 +61,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -72,6 +74,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -102,6 +105,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -156,15 +160,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -178,6 +195,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -186,6 +204,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -199,6 +218,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -207,6 +227,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -223,22 +244,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -249,6 +259,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -286,6 +297,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -328,6 +340,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -371,6 +384,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -379,6 +393,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -387,6 +402,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -415,6 +431,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -423,6 +441,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -436,6 +455,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -485,6 +505,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -515,6 +536,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -543,6 +565,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -579,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -587,6 +611,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -594,6 +620,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -602,6 +630,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml index 3a624b11..26b6ed6e 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml index bae736e2..9aeec323 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -113,6 +117,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true value: @@ -125,6 +130,8 @@ classes: range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true NWBContainer: name: NWBContainer @@ -134,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -145,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -155,6 +164,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -180,6 +190,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -188,6 +199,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -231,6 +243,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -241,6 +255,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -327,6 +342,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -358,6 +374,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -384,6 +401,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: @@ -413,4 +431,5 @@ classes: range: ImageReferences required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml index 0f6f89e0..9d963895 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml index 4dd254be..f41ac54f 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml index d63fc108..6fba3419 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml index fb0df61b..0a9685be 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,6 +65,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries annotations: @@ -77,6 +79,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -90,4 +93,5 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml index f5d5d497..481256fa 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -281,6 +292,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -376,6 +388,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -403,6 +416,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -419,18 +434,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -455,6 +476,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -474,6 +496,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -483,12 +506,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -497,6 +524,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -560,9 +588,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -617,6 +649,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -635,6 +668,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -644,6 +679,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -661,6 +698,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -669,6 +708,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -678,6 +719,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -687,6 +730,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -694,6 +739,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -703,6 +750,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -713,18 +761,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -732,6 +786,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -739,6 +795,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -749,6 +806,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: @@ -757,6 +815,7 @@ classes: range: Subject__age required: false multivalued: false + inlined: true date_of_birth: name: date_of_birth description: Date of birth of subject. Can be supplied instead of 'age'. @@ -815,6 +874,7 @@ classes: name: name: name ifabsent: string(age) + identifier: true range: string required: true equals_string: age diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml index b3181bc4..140e8c88 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -153,6 +160,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -189,6 +197,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -197,6 +206,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -205,6 +215,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -231,6 +242,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -239,48 +251,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -289,6 +309,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -313,6 +334,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -334,6 +356,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -355,6 +378,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -377,6 +401,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -399,6 +424,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -421,6 +447,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -443,6 +470,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -465,6 +493,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -473,6 +502,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -481,6 +511,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -505,6 +536,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true cell_id: @@ -565,6 +597,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -579,6 +612,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -593,9 +627,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -609,6 +647,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -617,6 +656,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -629,9 +669,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -640,6 +684,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -663,6 +708,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -671,6 +717,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -694,6 +741,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -713,6 +761,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -734,18 +783,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -757,6 +812,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -767,6 +823,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -780,6 +837,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -790,6 +848,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -800,6 +859,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -811,6 +871,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -821,6 +882,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -834,6 +896,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -853,6 +916,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -863,6 +927,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -874,6 +939,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -884,6 +950,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -897,6 +964,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -907,6 +975,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -917,6 +986,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -926,6 +996,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -935,6 +1006,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -948,6 +1020,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -957,6 +1030,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -967,3 +1041,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml index 45bd0a36..44062848 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -252,6 +287,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -272,6 +308,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -283,6 +320,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Images - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml index 56f88245..ced89853 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit in seconds. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml index 93ab4afb..b4858223 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml index 730ece09..3da9ec53 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -65,6 +66,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -76,6 +78,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -113,6 +116,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -125,6 +129,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -155,6 +160,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -209,15 +215,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -231,6 +250,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -239,6 +259,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -252,6 +273,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -260,6 +282,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -276,22 +299,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -302,6 +314,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -339,6 +352,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -381,6 +395,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -424,6 +439,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -432,6 +448,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -440,6 +457,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -468,6 +486,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -476,6 +496,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -489,6 +510,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -538,6 +560,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -568,6 +591,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -596,6 +620,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -632,6 +657,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -640,6 +666,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -647,6 +675,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -655,6 +685,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml index dc790f36..c1fce823 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml index ca7cfe13..7c3450a6 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -113,6 +117,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true value: @@ -125,6 +130,8 @@ classes: range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true NWBContainer: name: NWBContainer @@ -134,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -145,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -155,6 +164,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -180,6 +190,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -188,6 +199,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -231,6 +243,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -241,6 +255,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -327,6 +342,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -358,6 +374,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -384,6 +401,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: @@ -413,4 +431,5 @@ classes: range: ImageReferences required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml index f0b74b70..32ff4f8e 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml index ab2fc92f..3116431b 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml index f2be1a30..71eadb41 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml index 18850245..471b87a4 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,6 +65,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries annotations: @@ -77,6 +79,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -90,4 +93,5 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml index 1b56d9dc..a6b27f57 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -283,6 +294,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -378,6 +390,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -405,6 +418,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -421,18 +436,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -457,6 +478,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -476,6 +498,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -485,12 +508,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -499,6 +526,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -562,9 +590,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -619,6 +651,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -637,6 +670,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -646,6 +681,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -663,6 +700,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -671,6 +710,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -680,6 +721,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -689,6 +732,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -696,6 +741,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -705,6 +752,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -715,18 +763,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -734,6 +788,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -741,6 +797,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -751,6 +808,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: @@ -759,6 +817,7 @@ classes: range: Subject__age required: false multivalued: false + inlined: true date_of_birth: name: date_of_birth description: Date of birth of subject. Can be supplied instead of 'age'. @@ -817,6 +876,7 @@ classes: name: name: name ifabsent: string(age) + identifier: true range: string required: true equals_string: age diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml index 710ba366..a8662e71 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -153,6 +160,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -189,6 +197,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -197,6 +206,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -205,6 +215,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -231,6 +242,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -239,48 +251,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -289,6 +309,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -313,6 +334,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -334,6 +356,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -355,6 +378,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -377,6 +401,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -399,6 +424,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -421,6 +447,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -443,6 +470,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -465,6 +493,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -473,6 +502,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -481,6 +511,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -505,6 +536,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true cell_id: @@ -565,6 +597,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -579,6 +612,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -593,9 +627,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -609,6 +647,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -617,6 +656,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -629,9 +669,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -640,6 +684,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -663,6 +708,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true stimulus_template: name: stimulus_template annotations: @@ -677,6 +723,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -685,6 +732,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -708,6 +756,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -727,6 +776,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -748,18 +798,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -771,6 +827,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -781,6 +838,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -794,6 +852,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -804,6 +863,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -814,6 +874,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -825,6 +886,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -835,6 +897,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -848,6 +911,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -867,6 +931,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -877,6 +942,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -888,6 +954,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -898,6 +965,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -911,6 +979,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -921,6 +990,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -931,6 +1001,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -940,6 +1011,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -949,6 +1021,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -962,6 +1035,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -971,6 +1045,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -981,3 +1056,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml index cac5d739..603c3515 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -252,6 +287,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -272,6 +308,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -283,6 +320,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Images - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml index 9395fd91..b30070d4 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit in seconds. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml index 085004da..9cc7b0d5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -47,6 +48,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -58,6 +60,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -88,6 +91,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml index 8452b74d..b5d3676c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -65,6 +66,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -76,6 +78,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -113,6 +116,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -125,6 +129,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -155,6 +160,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -209,15 +215,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -231,6 +250,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -239,6 +259,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -252,6 +273,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -260,6 +282,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -276,22 +299,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -302,6 +314,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -339,6 +352,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -381,6 +395,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -424,6 +439,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -432,6 +448,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -440,6 +457,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -468,6 +486,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -476,6 +496,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -489,6 +510,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -538,6 +560,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -568,6 +591,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -596,6 +620,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -632,6 +657,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -640,6 +666,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -647,6 +675,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -655,6 +685,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml index 6416821d..8cc18100 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml index 27a272cb..a38fa1a2 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml @@ -86,6 +86,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml index fe82d7df..a668487e 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml @@ -86,6 +86,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml index 4285b03b..1022ebfa 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml @@ -114,6 +114,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml index 7746e8e5..e7f4d412 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml @@ -87,6 +87,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml index 2ce11ab6..b506697f 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml @@ -87,6 +87,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml index cae8e9ec..94bb9f7b 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml @@ -87,6 +87,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml index 266d216e..6ba8106e 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml index 13b5f58f..bbf0b5d1 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml index a88c85f6..7e1a614a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,4 +183,5 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml index 36edb0ab..91de7c24 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml index 24ea8fd2..b256bbeb 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml index 2652e1c6..f8adba6c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml index 8685dc75..4fd80e65 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml index 21654dfa..bde18eba 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml index cb79bbe8..52b119d9 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml index 5ba5b733..beb539c8 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml index 7ed736fb..6085aa3e 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml index 5240bc3a..85675e72 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml index 4ad36b72..f65f22b0 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml index 6167b42f..8974bc03 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml index c20fdb32..9ffb97d5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml index bb9b3240..ea83af31 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml index 842d1d67..9fd8ddd5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml index cf143215..940f1b7f 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml index 064f647d..2a10ba25 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml index 05dc8557..a962b8f3 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true resources: name: resources description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__resources required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,12 +54,14 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -66,6 +72,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -86,6 +93,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -130,6 +138,7 @@ classes: name: name: name ifabsent: string(resources) + identifier: true range: string required: true equals_string: resources @@ -158,6 +167,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -186,6 +196,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml index 94b31943..1f495086 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml index a1b6ec09..89023ae6 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true resources: name: resources description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__resources required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,12 +54,14 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -66,6 +72,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -86,6 +93,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -130,6 +138,7 @@ classes: name: name: name ifabsent: string(resources) + identifier: true range: string required: true equals_string: resources @@ -158,6 +167,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -198,6 +208,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml index 4991b33f..96f372c7 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml index ca25659f..d0909a2a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true files: name: files description: A table for storing object ids of files used in external resources. range: ExternalResources__files required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,12 +54,14 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -66,6 +72,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -86,6 +93,7 @@ classes: name: name: name ifabsent: string(files) + identifier: true range: string required: true equals_string: files @@ -106,6 +114,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -144,6 +153,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -201,6 +211,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml index 63329393..28ae7e8e 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml index e2acf658..75f3938d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true files: name: files description: A table for storing object ids of files used in external resources. range: ExternalResources__files required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,18 +54,21 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true entity_keys: name: entity_keys description: A table for identifying which keys use which entity. range: ExternalResources__entity_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -72,6 +79,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -92,6 +100,7 @@ classes: name: name: name ifabsent: string(files) + identifier: true range: string required: true equals_string: files @@ -112,6 +121,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -142,6 +152,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -199,6 +210,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys @@ -227,6 +239,7 @@ classes: name: name: name ifabsent: string(entity_keys) + identifier: true range: string required: true equals_string: entity_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml index c6cf1d48..208be725 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml index 7478fe17..dcaf9604 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -30,18 +31,21 @@ classes: range: HERD__keys required: true multivalued: false + inlined: true files: name: files description: A table for storing object ids of files used in external resources. range: HERD__files required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: HERD__entities required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -49,18 +53,21 @@ classes: range: HERD__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: HERD__object_keys required: true multivalued: false + inlined: true entity_keys: name: entity_keys description: A table for identifying which keys use which entity. range: HERD__entity_keys required: true multivalued: false + inlined: true tree_root: true HERD__keys: name: HERD__keys @@ -71,6 +78,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -91,6 +99,7 @@ classes: name: name: name ifabsent: string(files) + identifier: true range: string required: true equals_string: files @@ -111,6 +120,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -141,6 +151,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -198,6 +209,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys @@ -226,6 +238,7 @@ classes: name: name: name ifabsent: string(entity_keys) + identifier: true range: string required: true equals_string: entity_keys diff --git a/scripts/generate_core.py b/scripts/generate_core.py index 221aeaf1..4aeb21a1 100644 --- a/scripts/generate_core.py +++ b/scripts/generate_core.py @@ -171,17 +171,11 @@ def generate_versions( shutil.rmtree(tmp_dir / "linkml") shutil.rmtree(tmp_dir / "pydantic") - # import the most recent version of the schemaz we built - latest_version = sorted((pydantic_path / "core").glob("v*"), key=os.path.getmtime)[-1] - # make inits to use the schema! we don't usually do this in the # provider class because we directly import the files there. with open(pydantic_path / "__init__.py", "w") as initfile: initfile.write(" ") - with open(pydantic_path / "__init__.py", "w") as initfile: - initfile.write(f"from .pydantic.core.{latest_version.name}.namespace import *") - subprocess.run(["black", "."]) finally: @@ -228,6 +222,11 @@ def parser() -> ArgumentParser: ), action="store_true", ) + parser.add_argument( + "--debug", + help="Add annotations to generated schema that indicate how they were generated", + action="store_true", + ) parser.add_argument("--pdb", help="Launch debugger on an error", action="store_true") return parser @@ -235,6 +234,12 @@ def parser() -> ArgumentParser: def main(): args = parser().parse_args() + if args.debug: + os.environ["NWB_LINKML_DEBUG"] = "true" + else: + if "NWB_LINKML_DEBUG" in os.environ: + del os.environ["NWB_LINKML_DEBUG"] + tmp_dir = make_tmp_dir(clear=True) git_dir = tmp_dir / "git" git_dir.mkdir(exist_ok=True)