Fix PDFs with no paragraphs issue

Author: gabriel-piles

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "trainable-entity-extractor"
-version = "2025.4.24.1"
+version = "2025.4.24.2"
 description = "This tool is a trainable text/PDF to entity extractor"
 license = { file = "LICENSE" }
 authors = [{ name = "HURIDOCS" }]

src/multilingual_paragraph_extractor/domain/ParagraphsFromLanguage.py

@@ -111,7 +111,9 @@ def remove_duplicated_text(self):
 
             cleaned_paragraphs.append(paragraph)
 
-        cleaned_paragraphs.append(self.paragraphs[-1])
+        if self.paragraphs:
+            cleaned_paragraphs.append(self.paragraphs[-1])
+
         return cleaned_paragraphs
 
     def remove_headers_and_footers(self):
@@ -128,7 +130,7 @@ def is_top_or_bottom_of_page(paragraph: ParagraphFeatures, page_height: int):
 
     def find_headers_with_similarities(self):
         paragraphs_on_top = [x for x in self.paragraphs if self.is_top_or_bottom_of_page(x, self.paragraphs[0].page_height)]
-        pages_number = max([x.page_number for x in self.paragraphs])
+        pages_number = max([x.page_number for x in self.paragraphs]) if self.paragraphs else 1
         headers = {}
         for paragraph in paragraphs_on_top:
             found_match = False
@@ -259,7 +261,7 @@ def set_alignment_scores(self):
         unmatched_2 = set(range(len(self.paragraphs)))
 
         indexes_matching: dict[int, int] = dict()
-        scores: dict[(ParagraphFeatures, ParagraphFeatures), float] = dict()
+        scores: dict[tuple[ParagraphFeatures, ParagraphFeatures], float] = dict()
 
         for threshold in THRESHOLD:
             last_idx2_inserted = 0
@@ -313,6 +315,8 @@ def set_alignment_scores(self):
 
     def is_same_pdf(self):
         paragraph_count = len(self._main_language_paragraphs)
+        if not paragraph_count:
+            return True
         unmatched_paragraphs = [x for x in self._main_language_paragraphs if x not in self._alignment_scores]
         match_percentage = 100 * (paragraph_count - len(unmatched_paragraphs)) / paragraph_count
         return 50 < match_percentage
@@ -462,7 +466,7 @@ def is_aligned(self, main_language: "ParagraphsFromLanguage") -> bool:
         return len(self._aligned_paragraphs) == len(main_language.paragraphs)
 
     def remove_big_no_text_paragraphs(self):
-        threshold_area = 0.2 * self.paragraphs[0].page_width * self.paragraphs[0].page_height
+        threshold_area = 0.2 * self.paragraphs[0].page_width * self.paragraphs[0].page_height if self.paragraphs else 0
         fixed_paragraphs = list()
 
         for paragraph in self.paragraphs:

src/multilingual_paragraph_extractor/tests/test_align_paragraphs.py

@@ -64,6 +64,46 @@ def test_align_paragraphs_when_no_main_language(self):
         self.assertEqual("English text", paragraphs_from_languages[0].paragraphs[0].text_cleaned)
         self.assertEqual("French text", paragraphs_from_languages[1].paragraphs[0].text_cleaned)
 
+    def test_align_paragraphs_when_no_paragraph_in_one_language(self):
+        language_paragraph_1 = ParagraphsFromLanguage(language="en", paragraphs=[], is_main_language=True)
+
+        pdf_data_paragraphs_2 = ParagraphFeatures.from_texts(texts=["French text"])
+        language_paragraph_2 = ParagraphsFromLanguage(
+            language="fr", paragraphs=pdf_data_paragraphs_2, is_main_language=False
+        )
+
+        multilingual_paragraph_extractor = MultilingualParagraphAlignerUseCase(
+            extractor_identifier=self.extraction_identifier
+        )
+        paragraphs_from_languages = [language_paragraph_1, language_paragraph_2]
+        multilingual_paragraph_extractor.align_languages(paragraphs_from_languages)
+
+        self.assertEqual(2, len(paragraphs_from_languages))
+        self.assertEqual(0, len(paragraphs_from_languages[0].paragraphs))
+        self.assertEqual(0, len(paragraphs_from_languages[1].paragraphs))
+
+    def test_align_paragraphs_when_no_paragraph_in_other_language(self):
+        pdf_data_paragraphs_1 = ParagraphFeatures.from_texts(texts=["English text"])
+        language_paragraph_1 = ParagraphsFromLanguage(language="en", paragraphs=pdf_data_paragraphs_1, is_main_language=True)
+
+        language_paragraph_2 = ParagraphsFromLanguage(language="fr", paragraphs=[], is_main_language=False)
+
+        multilingual_paragraph_extractor = MultilingualParagraphAlignerUseCase(
+            extractor_identifier=self.extraction_identifier
+        )
+        paragraphs_from_languages = [language_paragraph_1, language_paragraph_2]
+        multilingual_paragraph_extractor.align_languages(paragraphs_from_languages)
+
+        self.assertEqual(2, len(paragraphs_from_languages))
+        self.assertEqual(1, len(paragraphs_from_languages[0].paragraphs))
+        self.assertEqual(1, len(paragraphs_from_languages[1].paragraphs))
+
+        self.assertEqual("en", paragraphs_from_languages[0].language)
+        self.assertEqual("fr", paragraphs_from_languages[1].language)
+
+        self.assertEqual("English text", paragraphs_from_languages[0].paragraphs[0].text_cleaned)
+        self.assertEqual("", paragraphs_from_languages[1].paragraphs[0].text_cleaned)
+
     @staticmethod
     def get_paragraphs(language: str):
         paragraphs = ParagraphFeatures.from_texts(texts=[f"a 0. {language}", f"b 1: {language}", f"c 2! {language}"])

src/multilingual_paragraph_extractor/use_cases/MultilingualParagraphAlignerUseCase.py

@@ -8,7 +8,7 @@ def __init__(self, extractor_identifier: ExtractionIdentifier):
 
     def align_languages(self, paragraphs_from_languages: list[ParagraphsFromLanguage]):
         if not paragraphs_from_languages:
-            return []
+            return
 
         self.clean_paragraphs(paragraphs_from_languages)
         main_language, other_languages = self.get_main_and_other_languages(paragraphs_from_languages)