Merge pull request #10 from p2p-ld/nwb-loader

NWB Loader

.github/workflows/tests.yml

@@ -36,7 +36,10 @@ jobs:
             nwb_models/pyproject.toml
 
       - name: Install dependencies
-        run: pip install -e .[tests]
+        run: |
+          pip install -e .[tests]
+          pip install -e ../nwb_schema_language
+          pip install -e ../nwb_models
         working-directory: nwb_linkml
 
       - name: Run Tests

docs/meta/todo.md

@@ -49,6 +49,10 @@ Remove monkeypatches/overrides once PRs are closed
 Tests
 - [ ] Ensure schemas and pydantic modules in repos are up to date
 
+Loading
+- [ ] Top-level containers are still a little janky, eg. how `ProcessingModule` just accepts 
+  extra args rather than properly abstracting `value` as a `__getitem__(self, key) -> T:` 
+
 ## Docs TODOs
 
 ```{todolist}

nwb_linkml/conftest.py

@@ -71,7 +71,7 @@ def parse_adapter_blocks(document: Document) -> Generator[Region, None, None]:
 
 doctest_parser = Sybil(
     parsers=[DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), PythonCodeBlockParser()],
-    patterns=["*.py"],
+    patterns=["providers/git.py"],
 )
 
 pytest_collect_file = (adapter_parser + doctest_parser).pytest()

nwb_linkml/pyproject.toml

@@ -9,7 +9,7 @@ license = {text = "AGPL-3.0"}
 readme = "README.md"
 requires-python = "<3.13,>=3.10"
 dependencies = [
-    "nwb-models>=0.1.0",
+    "nwb-models>=0.2.0",
     "pyyaml>=6.0",
     "linkml-runtime>=1.7.7",
     "nwb-schema-language>=0.1.3",
@@ -22,9 +22,10 @@ dependencies = [
     "pydantic-settings>=2.0.3",
     "tqdm>=4.66.1",
     'typing-extensions>=4.12.2;python_version<"3.11"',
-    "numpydantic>=1.3.3",
+    "numpydantic>=1.5.0",
     "black>=24.4.2",
     "pandas>=2.2.2",
+    "networkx>=3.3",
 ]
 
 [project.urls]
@@ -44,6 +45,7 @@ tests = [
     "pytest-cov<5.0.0,>=4.1.0",
     "sybil>=6.0.3",
     "requests-cache>=1.2.1",
+    "pynwb>=2.8.1",
 ]
 dev = [
     "nwb-linkml[tests]",

nwb_linkml/src/nwb_linkml/adapters/adapter.py

@@ -2,6 +2,7 @@
 Base class for adapters
 """
 
+import os
 import sys
 from abc import abstractmethod
 from dataclasses import dataclass, field
@@ -101,6 +102,19 @@ class Adapter(BaseModel):
     """Abstract base class for adapters"""
 
     _logger: Optional[Logger] = None
+    _debug: Optional[bool] = None
+
+    @property
+    def debug(self) -> bool:
+        """
+        Whether we are in debug mode, which adds extra metadata in generated elements.
+
+        Set explicitly via ``_debug`` , or else checks for the truthiness of the
+        environment variable ``NWB_LINKML_DEBUG``
+        """
+        if self._debug is None:
+            self._debug = bool(os.environ.get("NWB_LINKML_DEBUG", False))
+        return self._debug
 
     @property
     def logger(self) -> Logger:

nwb_linkml/src/nwb_linkml/adapters/attribute.py

@@ -10,7 +10,7 @@
 from nwb_linkml.adapters.adapter import Adapter, BuildResult, is_1d
 from nwb_linkml.adapters.array import ArrayAdapter
 from nwb_linkml.maps import Map
-from nwb_linkml.maps.dtype import handle_dtype
+from nwb_linkml.maps.dtype import handle_dtype, inlined
 from nwb_schema_language import Attribute
 
 
@@ -104,6 +104,7 @@ def apply(cls, attr: Attribute, res: Optional[BuildResult] = None) -> BuildResul
             range=handle_dtype(attr.dtype),
             description=attr.doc,
             required=attr.required,
+            inlined=inlined(attr.dtype),
             **cls.handle_defaults(attr),
         )
         return BuildResult(slots=[slot])
@@ -151,6 +152,7 @@ def apply(cls, attr: Attribute, res: Optional[BuildResult] = None) -> BuildResul
             multivalued=multivalued,
             description=attr.doc,
             required=attr.required,
+            inlined=inlined(attr.dtype),
             **expressions,
             **cls.handle_defaults(attr),
         )
@@ -171,7 +173,10 @@ def build(self) -> "BuildResult":
         Build the slot definitions, every attribute should have a map.
         """
         map = self.match()
-        return map.apply(self.cls)
+        res = map.apply(self.cls)
+        if self.debug:  # pragma: no cover - only used in development
+            res = self._amend_debug(res, map)
+        return res
 
     def match(self) -> Optional[Type[AttributeMap]]:
         """
@@ -195,3 +200,13 @@ def match(self) -> Optional[Type[AttributeMap]]:
             return None
         else:
             return matches[0]
+
+    def _amend_debug(
+        self, res: BuildResult, map: Optional[Type[AttributeMap]] = None
+    ) -> BuildResult:  # pragma: no cover - only used in development
+        map_name = "None" if map is None else map.__name__
+        for cls in res.classes:
+            cls.annotations["attribute_map"] = {"tag": "attribute_map", "value": map_name}
+        for slot in res.slots:
+            slot.annotations["attribute_map"] = {"tag": "attribute_map", "value": map_name}
+        return res

nwb_linkml/src/nwb_linkml/adapters/classes.py

@@ -92,6 +92,13 @@ def build_base(self, extra_attrs: Optional[List[SlotDefinition]] = None) -> Buil
         # Get vanilla top-level attributes
         kwargs["attributes"].extend(self.build_attrs(self.cls))
 
+        if self.debug:  # pragma: no cover - only used in development
+            kwargs["annotations"] = {}
+            kwargs["annotations"]["group_adapter"] = {
+                "tag": "group_adapter",
+                "value": "container_slot",
+            }
+
         if extra_attrs is not None:
             if isinstance(extra_attrs, SlotDefinition):
                 extra_attrs = [extra_attrs]
@@ -230,18 +237,23 @@ def build_name_slot(self) -> SlotDefinition:
                 ifabsent=f"string({name})",
                 equals_string=equals_string,
                 range="string",
+                identifier=True,
             )
         else:
-            name_slot = SlotDefinition(name="name", required=True, range="string")
+            name_slot = SlotDefinition(name="name", required=True, range="string", identifier=True)
         return name_slot
 
     def build_self_slot(self) -> SlotDefinition:
         """
         If we are a child class, we make a slot so our parent can refer to us
         """
-        return SlotDefinition(
+        slot = SlotDefinition(
             name=self._get_slot_name(),
             description=self.cls.doc,
             range=self._get_full_name(),
+            inlined=True,
             **QUANTITY_MAP[self.cls.quantity],
         )
+        if self.debug:  # pragma: no cover - only used in development
+            slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "self_slot"}
+        return slot