Fix numpy array concatenation

Author: gabriel-piles

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "trainable-entity-extractor"
-version = "2025.03.10"
+version = "2025.03.11"
 description = "This tool is a trainable text/PDF to entity extractor"
 license = { file = "LICENSE" }
 authors = [{ name = "HURIDOCS" }]

src/trainable_entity_extractor/use_cases/extractors/pdf_to_multi_option_extractor/filter_segments_methods/Beginning750.py

@@ -1,4 +1,5 @@
 from copy import deepcopy
+from typing import Optional
 
 from trainable_entity_extractor.domain.PdfDataSegment import PdfDataSegment
 from trainable_entity_extractor.use_cases.extractors.pdf_to_multi_option_extractor.FilterSegmentsMethod import (
@@ -22,7 +23,7 @@ def get_first_tokens(self, pdf_data_segments: list[PdfDataSegment], text_length:
         return filtered_segments
 
     @staticmethod
-    def get_segment(pdf_data_segment: PdfDataSegment, character_limit: int):
+    def get_segment(pdf_data_segment: PdfDataSegment, character_limit: int) -> Optional[PdfDataSegment]:
         if character_limit <= 0:
             return None
 

src/trainable_entity_extractor/use_cases/extractors/pdf_to_multi_option_extractor/filter_segments_methods/CleanEndDot500.py

@@ -18,7 +18,7 @@ def get_tokens(self, pdf_data_segments: list[PdfDataSegment], text_length: int)
             if not pdf_data_segment_copy:
                 break
 
-            if pdf_data_segment_copy.text_content and "." == pdf_data_segment.text_cleaned[-1]:
+            if pdf_data_segment_copy.text_content and "." == pdf_data_segment.text_content[-1]:
                 pdf_data_segment_copy.text_content += "."
 
             total_text += " " + pdf_data_segment_copy.text_content

src/trainable_entity_extractor/use_cases/extractors/pdf_to_multi_option_extractor/filter_segments_methods/End750.py

@@ -14,7 +14,7 @@ def get_first_tokens(self, pdf_data_segments: list[PdfDataSegment], text_length:
             if not pdf_data_segment_copy:
                 break
 
-            total_text += " " + pdf_data_segment_copy.text_cleaned
+            total_text += " " + pdf_data_segment_copy.text_content
             filtered_segments.append(pdf_data_segment_copy)
 
         return list(reversed(filtered_segments))

src/trainable_entity_extractor/use_cases/extractors/pdf_to_multi_option_extractor/multi_option_extraction_methods/SentenceSelectorFuzzyCommas.py

@@ -42,7 +42,7 @@ def get_sentence_segment_list(self, pdf_data_segments) -> list[(str, PdfDataSegm
 
         sentence_segment_list = []
         for segment in merged_sentences:
-            segment_text = " ".join(segment.text_cleaned.split())
+            segment_text = " ".join(segment.text_content.split())
             for text in re.split(r"\.|:", segment_text):
                 if not text:
                     continue
@@ -65,17 +65,17 @@ def get_sentence_segment_list(self, pdf_data_segments) -> list[(str, PdfDataSegm
         return sentences_across_pages
 
     def get_segments_merged(self, segments):
-        segments = [segment for segment in segments if segment.text_cleaned.strip()]
+        segments = [segment for segment in segments if segment.text_content.strip()]
         if not segments:
             return list()
 
         merged_sentences = [segments[0]]
         for segment in segments[1:]:
-            previous_segment_text = " ".join(merged_sentences[-1].text_cleaned.split())
+            previous_segment_text = " ".join(merged_sentences[-1].text_content.split())
 
             if previous_segment_text[-1] not in [".", ":"]:
                 merged_segment = deepcopy(merged_sentences[-1])
-                merged_segment.text_cleaned = f"{previous_segment_text}, {' '.join(segment.text_cleaned.split())}"
+                merged_segment.text_content = f"{previous_segment_text}, {' '.join(segment.text_content.split())}"
                 bounding_boxes = [merged_segment.bounding_box, segment.bounding_box]
                 merged_segment.bounding_box = Rectangle.merge_rectangles(bounding_boxes)
                 merged_sentences[-1] = merged_segment
@@ -89,7 +89,7 @@ def get_sample(sample: TrainingSample, sentence_segment_list: list[(str, PdfData
         sentence_segments = list()
         for sentence, segment in sentence_segment_list:
             sentence_segment = deepcopy(segment)
-            sentence_segment.text_cleaned = sentence
+            sentence_segment.text_content = sentence
             sentence_segments.append(sentence_segment)
 
         sentence_pdf_data = PdfData(pdf_features=None, file_name=sample.pdf_data.file_name)

src/trainable_entity_extractor/use_cases/extractors/pdf_to_text_extractor/methods/GlinerFirstDateMethod.py

@@ -17,10 +17,10 @@ def get_date_from_segments(self, segments: list[PdfDataSegment], languages):
         merge_segments: list[list[PdfDataSegment]] = self.merge_segments_for_dates(segments)
         for segments in merge_segments:
             segment_merged = PdfDataSegment.from_list_to_merge(segments)
-            if not self.contains_year(segment_merged.text_cleaned):
+            if not self.contains_year(segment_merged.text_content):
                 continue
 
-            date = GlinerDateParserMethod.get_date([segment_merged.text_cleaned])
+            date = GlinerDateParserMethod.get_date([segment_merged.text_content])
             if date:
                 for segment in segments:
                     segment.ml_label = 1

src/trainable_entity_extractor/use_cases/extractors/pdf_to_text_extractor/methods/SpaceFixerGlinerFirstDateMethod.py

@@ -15,9 +15,9 @@ def contains_year(text: str):
 
     def get_date_from_segments(self, segments: list[PdfDataSegment], languages: list[str]) -> str:
         for segment in self.loop_segments(segments):
-            if not self.contains_year(segment.text_cleaned):
+            if not self.contains_year(segment.text_content):
                 continue
-            date = GlinerDateParserMethod.get_date([segment.text_cleaned])
+            date = GlinerDateParserMethod.get_date([segment.text_content])
             if date:
                 segment.ml_label = 1
                 return date.strftime("%Y-%m-%d")

src/trainable_entity_extractor/use_cases/extractors/segment_selector/FastSegmentSelector.py

@@ -150,8 +150,8 @@ def predict(self, segments):
             return []
 
         model = lgb.Booster(model_file=self.model_path)
-        predictions = model.predict(x)
-
+        predictions_array = model.predict(x)
+        predictions = list(predictions_array) if predictions_array is not None else []
         return self.predictions_scores_to_segments(segments, predictions)
 
     def predictions_scores_to_segments(self, segments: list[PdfDataSegment], prediction_scores: list[float]):

src/trainable_entity_extractor/use_cases/extractors/segment_selector/methods/avoiding_words/AvoidingWords.py

@@ -109,8 +109,8 @@ def save_most_frequent_words(self, model_path):
 
         avoiding_words = Counter()
         for segment in [segment for segment in self.segments if segment.pdf_segment.ml_label]:
-            text_tokens = word_tokenize(segment.next_segment.text_cleaned) if segment.next_segment else []
-            text_tokens += word_tokenize(segment.previous_segment.text_cleaned) if segment.previous_segment else []
+            text_tokens = word_tokenize(segment.next_segment.text_content) if segment.next_segment else []
+            text_tokens += word_tokenize(segment.previous_segment.text_content) if segment.previous_segment else []
             avoiding_words.update(
                 [
                     word.lower()