#135 removed. #131 simplified NER embedding into AREkit pipelines.

arelight/pipelines/items/entities_ner_dp.py

@@ -1,5 +1,5 @@
 from arekit.common.bound import Bound
-from arekit.common.docs.objects_parser import SentenceObjectsParserPipelineItem
+from arekit.common.pipeline.items.base import BasePipelineItem
 from arekit.common.text.partitioning.terms import TermsPartitioning
 
 from arelight.ner.deep_pavlov import DeepPavlovNER
@@ -8,7 +8,7 @@
 from arelight.utils import IdAssigner
 
 
-class DeepPavlovNERPipelineItem(SentenceObjectsParserPipelineItem):
+class DeepPavlovNERPipelineItem(BasePipelineItem):
 
  def __init__(self, id_assigner, ner_model_name, obj_filter=None,
  chunk_limit=128, display_value_func=None, **kwargs):
@@ -19,17 +19,15 @@ def __init__(self, id_assigner, ner_model_name, obj_filter=None,
  assert(isinstance(chunk_limit, int) and chunk_limit > 0)
  assert(isinstance(id_assigner, IdAssigner))
  assert(callable(display_value_func) or display_value_func is None)
- super(DeepPavlovNERPipelineItem, self).__init__(partitioning=TermsPartitioning(), **kwargs)
+ super(DeepPavlovNERPipelineItem, self).__init__(**kwargs)
 
  # Initialize bert-based model instance.
  self.__dp_ner = DeepPavlovNER(ner_model_name)
  self.__obj_filter = obj_filter
  self.__chunk_limit = chunk_limit
  self.__id_assigner = id_assigner
  self.__disp_value_func = display_value_func
-
- def _get_parts_provider_func(self, input_data):
- return self.__iter_subs_values_with_bounds(input_data)
+ self.__partitioning = TermsPartitioning()
 
  def __iter_subs_values_with_bounds(self, terms_list):
  assert(isinstance(terms_list, list))
@@ -66,3 +64,7 @@ def __iter_parsed_entities(self, processed_sequences, chunk_terms_list, chunk_of
  value=value, e_type=s_obj.ObjectType, entity_id=self.__id_assigner.get_id(),
  display_value=self.__disp_value_func(value) if self.__disp_value_func is not None else None)
  yield entity, Bound(pos=chunk_offset + s_obj.Position, length=s_obj.Length)
+
+ def apply_core(self, input_data, pipeline_ctx):
+ parts_it = self.__iter_subs_values_with_bounds(input_data)
+ return self.__partitioning.provide(text=input_data, parts_it=parts_it)

arelight/pipelines/items/entities_ner_transformers.py

@@ -1,11 +1,13 @@
 from arekit.common.bound import Bound
-from arekit.common.docs.objects_parser import SentenceObjectsParserPipelineItem
+from arekit.common.pipeline.items.base import BasePipelineItem
+from arekit.common.text.partitioning.str import StringPartitioning
+from arekit.common.utils import split_by_whitespaces
 
 from arelight.pipelines.items.entity import IndexedEntity
 from arelight.utils import IdAssigner, auto_import
 
 
-class TransformersNERPipelineItem(SentenceObjectsParserPipelineItem):
+class TransformersNERPipelineItem(BasePipelineItem):
 
  def __init__(self, id_assigner, ner_model_name, device, obj_filter=None, display_value_func=None, **kwargs):
  """ chunk_limit: int
@@ -28,8 +30,11 @@ def __init__(self, id_assigner, ner_model_name, device, obj_filter=None, display
  self.__obj_filter = obj_filter
  self.__id_assigner = id_assigner
  self.__disp_value_func = display_value_func
+ self.__partitioning = StringPartitioning()
 
- def _get_parts_provider_func(self, input_data):
+ # region Private methods
+
+ def __get_parts_provider_func(self, input_data):
  assert(isinstance(input_data, str))
  parts = self.annotate_ner(model=self.__model, tokenizer=self.__tokenizer, text=input_data,
  device=self.__device)
@@ -52,3 +57,19 @@ def __iter_parsed_entities(self, parts):
  display_value=self.__disp_value_func(value) if self.__disp_value_func is not None else None)
 
  yield entity, Bound(pos=p["start"], length=p["end"] - p["start"])
+
+ @staticmethod
+ def __iter_fixed_terms(terms):
+ for e in terms:
+ if isinstance(e, str):
+ for term in split_by_whitespaces(e):
+ yield term
+ else:
+ yield e
+
+ # endregion
+
+ def apply_core(self, input_data, pipeline_ctx):
+ parts_it = self.__get_parts_provider_func(input_data)
+ handled = self.__partitioning.provide(text=input_data, parts_it=parts_it)
+ return list(self.__iter_fixed_terms(handled))

arelight/run/infer.py

@@ -31,7 +31,6 @@
 from arelight.pipelines.items.entities_default import TextEntitiesParser
 from arelight.pipelines.items.entities_ner_dp import DeepPavlovNERPipelineItem
 from arelight.pipelines.items.entities_ner_transformers import TransformersNERPipelineItem
-from arelight.pipelines.items.terms_splitter import CustomTermsSplitterPipelineItem
 from arelight.run.utils import merge_dictionaries, iter_group_values, create_sentence_parser, \
  create_translate_model, is_port_number, iter_content, OPENNRE_CHECKPOINTS
 from arelight.samplers.bert import create_bert_sample_provider
@@ -242,7 +241,6 @@ def __entity_display_value(entity_value):
  text_parser_pipeline = [
  BasePipelineItem(src_func=lambda s: s.Text),
  entity_parsers[ner_framework](),
- CustomTermsSplitterPipelineItem() if ner_framework == "transformers" else None,
  text_translator_setup["ml-based" if args.translate_text is not None else None](),
  EntitiesGroupingPipelineItem(
  lambda value: SynonymsCollectionValuesGroupingProviders.provide_existed_or_register_missed_value(

test/test_transformers_ner_ppl.py

@@ -3,12 +3,10 @@
 from tqdm import tqdm
 
 from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.text.partitioning.str import StringPartitioning
 from arekit.common.utils import split_by_whitespaces
 
 from arelight.pipelines.items.entities_ner_dp import DeepPavlovNERPipelineItem
 from arelight.pipelines.items.entities_ner_transformers import TransformersNERPipelineItem
-from arelight.pipelines.items.terms_splitter import CustomTermsSplitterPipelineItem
 from arelight.third_party.transformers import annotate_ner_ppl, init_token_classification_model, annotate_ner
 from arelight.utils import IdAssigner
 from utils_ner import test_ner
@@ -50,9 +48,9 @@ def test_transformers(self):
  # Declare input texts.
 
  ppl_items = [
- TransformersNERPipelineItem(id_assigner=IdAssigner(), ner_model_name="dslim/bert-base-NER", device="cpu",
- src_func=lambda s: s.Text, partitioning=StringPartitioning()),
- CustomTermsSplitterPipelineItem(),
+ TransformersNERPipelineItem(id_assigner=IdAssigner(),
+ ner_model_name="dslim/bert-base-NER", device="cpu",
+  src_func=lambda s: s.Text)
  ]
 
  test_ner(texts=self.get_texts(), ner_ppl_items=ppl_items, prefix="transformers-ner")

test/utils_ner.py

@@ -66,4 +66,4 @@ def test_ner(texts, ner_ppl_items, prefix):
  "data_type_pipelines": {DataType.Test: test_pipeline}
  })
 
- BasePipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key="doc_ids")
+ BasePipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key="doc_ids")