feat: add option to skip table chunking (#4338)

This PR adds a new option to chunker `skip_table_chunking`: when `True`
table elements are not split by the chunking process, default to `False`
(current behavior).

---------

Co-authored-by: cragwolfe <crag@unstructured.io>

Author: badGarnet

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.22.21
+
+### Enhancements
+
+- **Skip table chunking option**: Add `skip_table_chunking` to basic/title chunking options. When `True`, `Table` elements are passed through unchanged without being split into `TableChunk` elements, regardless of their size. Defaults to `False` to preserve existing behavior.
+
 ## 0.22.20
 
 ### Enhancements

test_unstructured/chunking/test_base.py

@@ -115,6 +115,18 @@ def it_knows_whether_to_repeat_table_headers_by_default(
     ):
         assert ChunkingOptions(**kwargs).repeat_table_headers is expected_value
 
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"skip_table_chunking": True}, True),
+            ({"skip_table_chunking": False}, False),
+            ({"skip_table_chunking": None}, False),
+            ({}, False),
+        ],
+    )
+    def it_knows_whether_to_skip_table_chunking(self, kwargs: dict[str, Any], expected_value: bool):
+        assert ChunkingOptions(**kwargs).skip_table_chunking is expected_value
+
     @pytest.mark.parametrize("n_chars", [-1, -42])
     def it_rejects_new_after_n_chars_for_n_less_than_zero(self, n_chars: int):
         with pytest.raises(
@@ -694,6 +706,29 @@ def it_can_chunk_elements_with_none_text_without_error(self):
         assert len(chunks) == 1
         assert chunks[0].text == "hello world"
 
+    def it_yields_an_oversized_table_unchanged_when_skip_table_chunking_is_True(self):
+        table_text = "cell " * 200  # 1000 chars, well above default max_characters=500
+        table = Table(table_text.strip())
+        opts = ChunkingOptions(max_characters=100, skip_table_chunking=True)
+        pre_chunk = PreChunk([table], overlap_prefix="", opts=opts)
+
+        chunks = list(pre_chunk.iter_chunks())
+
+        assert len(chunks) == 1
+        assert isinstance(chunks[0], Table)
+        assert chunks[0] is table
+
+    def it_splits_an_oversized_table_when_skip_table_chunking_is_False(self):
+        table_text = "cell " * 200  # 1000 chars, well above max_characters=100
+        table = Table(table_text.strip())
+        opts = ChunkingOptions(max_characters=100, skip_table_chunking=False)
+        pre_chunk = PreChunk([table], overlap_prefix="", opts=opts)
+
+        chunks = list(pre_chunk.iter_chunks())
+
+        assert len(chunks) > 1
+        assert all(isinstance(c, TableChunk) for c in chunks)
+
     @pytest.mark.parametrize(
         ("max_characters", "combine_text_under_n_chars", "expected_value"),
         [

test_unstructured/chunking/test_basic.py

@@ -183,6 +183,44 @@ def test_it_repeats_table_headers_by_default_but_can_opt_out():
     ]
 
 
+def test_skip_table_chunking_passes_oversized_table_through_unchanged():
+    table_text = "cell " * 200  # 1000 chars, well above max_characters=100
+    table = Table(table_text.strip())
+    text_before = Text("Hello world")
+    text_after = Text("Goodbye world")
+
+    chunks = chunk_elements(
+        [text_before, table, text_after],
+        max_characters=100,
+        skip_table_chunking=True,
+    )
+
+    assert len(chunks) == 3
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+    assert isinstance(chunks[2], CompositeElement)
+    # -- table text is unchanged --
+    assert chunks[1].text == table_text.strip()
+
+
+def test_skip_table_chunking_does_not_affect_text_element_chunking():
+    long_text = Text("word " * 200)
+    table = Table("small table")
+
+    chunks = chunk_elements(
+        [long_text, table],
+        max_characters=100,
+        skip_table_chunking=True,
+    )
+
+    # -- long text element is still split, table is still isolated --
+    text_chunks = [c for c in chunks if isinstance(c, CompositeElement)]
+    table_chunks = [c for c in chunks if isinstance(c, Table)]
+    assert len(text_chunks) > 1
+    assert len(table_chunks) == 1
+    assert table_chunks[0].text == "small table"
+
+
 # ------------------------------------------------------------------------------------------------
 # UNIT TESTS
 # ------------------------------------------------------------------------------------------------
@@ -229,6 +267,23 @@ def it_supports_the_repeat_table_headers_option(
         _, opts = _chunk_elements_.call_args.args
         assert opts.repeat_table_headers is expected_value
 
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"skip_table_chunking": True}, True),
+            ({"skip_table_chunking": False}, False),
+            ({"skip_table_chunking": None}, False),
+            ({}, False),
+        ],
+    )
+    def it_supports_the_skip_table_chunking_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.skip_table_chunking is expected_value
+
     # -- fixtures --------------------------------------------------------------------------------
 
     @pytest.fixture()

test_unstructured/chunking/test_title.py

@@ -65,6 +65,34 @@ def test_it_splits_oversized_table():
     assert all(isinstance(chunk, TableChunk) for chunk in chunks)
 
 
+def test_skip_table_chunking_passes_oversized_table_through_unchanged():
+    elements = elements_from_json(input_path("chunking/table_2000.json"))
+
+    chunks = chunk_by_title(elements, skip_table_chunking=True)
+
+    assert len(chunks) == 1
+    assert isinstance(chunks[0], Table)
+
+
+def test_skip_table_chunking_does_not_combine_table_with_adjacent_text():
+    table_text = "cell " * 200
+    table = Table(table_text.strip())
+    text_before = Text("Hello world")
+    text_after = Text("Goodbye world")
+
+    chunks = chunk_by_title(
+        [text_before, table, text_after],
+        max_characters=5000,
+        combine_text_under_n_chars=5000,
+        skip_table_chunking=True,
+    )
+
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+    assert isinstance(chunks[2], CompositeElement)
+    assert chunks[1].text == table_text.strip()
+
+
 def test_it_repeats_table_headers_by_default_but_can_opt_out():
     table_html = (
         "<table>"
@@ -564,6 +592,23 @@ def it_supports_the_repeat_table_headers_option(
         _, opts = _chunk_by_title_.call_args.args
         assert opts.repeat_table_headers is expected_value
 
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"skip_table_chunking": True}, True),
+            ({"skip_table_chunking": False}, False),
+            ({"skip_table_chunking": None}, False),
+            ({}, False),
+        ],
+    )
+    def it_supports_the_skip_table_chunking_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.skip_table_chunking is expected_value
+
     # -- fixtures --------------------------------------------------------------------------------
 
     @pytest.fixture()

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.22.20"  # pragma: no cover
+__version__ = "0.22.21"  # pragma: no cover

unstructured/chunking/base.py

@@ -199,6 +199,15 @@ def repeat_table_headers(self) -> bool:
         arg_value = self._kwargs.get("repeat_table_headers")
         return True if arg_value is None else bool(arg_value)
 
+    @cached_property
+    def skip_table_chunking(self) -> bool:
+        """When True, Table elements are passed through without chunking.
+
+        Default value is `False`.
+        """
+        arg_value = self._kwargs.get("skip_table_chunking")
+        return False if arg_value is None else bool(arg_value)
+
     @cached_property
     def inter_chunk_overlap(self) -> int:
         """Characters of overlap to add between chunks.
@@ -662,9 +671,12 @@ def iter_chunks(self) -> Iterator[CompositeElement | Table | TableChunk]:
         # -- it may need to be split into multiple `TableChunk` elements and that operation is
         # -- quite specialized.
         if len(self._elements) == 1 and isinstance(self._elements[0], Table):
-            yield from _TableChunker.iter_chunks(
-                self._elements[0], self._overlap_prefix, self._opts
-            )
+            if self._opts.skip_table_chunking:
+                yield self._elements[0]
+            else:
+                yield from _TableChunker.iter_chunks(
+                    self._elements[0], self._overlap_prefix, self._opts
+                )
         else:
             yield from _Chunker.iter_chunks(self._elements, self._text, self._opts)
 

unstructured/chunking/basic.py

@@ -33,6 +33,7 @@ def chunk_elements(
     overlap_all: Optional[bool] = None,
     tokenizer: Optional[str] = None,
     repeat_table_headers: Optional[bool] = None,
+    skip_table_chunking: Optional[bool] = None,
 ) -> list[Element]:
     """Combine sequential `elements` into chunks, respecting specified text-length limits.
 
@@ -80,6 +81,9 @@ def chunk_elements(
     repeat_table_headers
         Default: `True`. When `True`, repeated table-header behavior is enabled for chunked table
         continuations. Specify `False` to opt out and preserve legacy table-chunk behavior.
+    skip_table_chunking
+        Default: `False`. When `True`, `Table` elements are passed through unchanged without
+        being split into `TableChunk` elements, regardless of their size.
     """
     # -- raises ValueError on invalid parameters --
     opts = _BasicChunkingOptions.new(
@@ -92,6 +96,7 @@ def chunk_elements(
         overlap_all=overlap_all,
         tokenizer=tokenizer,
         repeat_table_headers=repeat_table_headers,
+        skip_table_chunking=skip_table_chunking,
     )
 
     return _chunk_elements(elements, opts)

unstructured/chunking/dispatch.py

@@ -71,6 +71,9 @@ def add_chunking_strategy(func: Callable[_P, list[Element]]) -> Callable[_P, lis
             + "\n\t\trepeat_table_headers"
             + "\n\t\t\tDefault: True. Repeat detected table headers on continuation"
             + "\n\t\t\ttable chunks. Set to False to opt out."
+            + "\n\t\tskip_table_chunking"
+            + "\n\t\t\tDefault: False. When True, Table elements are passed through"
+            + "\n\t\t\tunchanged without being split into TableChunk elements."
         )
 
     @functools.wraps(func)

unstructured/chunking/title.py

@@ -34,6 +34,7 @@ def chunk_by_title(
     overlap_all: Optional[bool] = None,
     tokenizer: Optional[str] = None,
     repeat_table_headers: Optional[bool] = None,
+    skip_table_chunking: Optional[bool] = None,
 ) -> list[Element]:
     """Uses title elements to identify sections within the document for chunking.
 
@@ -87,6 +88,9 @@ def chunk_by_title(
     repeat_table_headers
         Default: `True`. When `True`, repeated table-header behavior is enabled for chunked table
         continuations. Specify `False` to opt out and preserve legacy table-chunk behavior.
+    skip_table_chunking
+        Default: `False`. When `True`, `Table` elements are passed through unchanged without
+        being split into `TableChunk` elements, regardless of their size.
     """
     opts = _ByTitleChunkingOptions.new(
         combine_text_under_n_chars=combine_text_under_n_chars,
@@ -100,6 +104,7 @@ def chunk_by_title(
         overlap_all=overlap_all,
         tokenizer=tokenizer,
         repeat_table_headers=repeat_table_headers,
+        skip_table_chunking=skip_table_chunking,
     )
     return _chunk_by_title(elements, opts)