#131 removed duplicated code parts

test/test_pipeline_infer.py

@@ -14,9 +14,7 @@
 from arekit.common.data import const
 from arekit.common.pipeline.context import PipelineContext
 from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.common.docs.base import Document
 from arekit.common.docs.entities_grouping import EntitiesGroupingPipelineItem
-from arekit.common.docs.sentence import BaseDocumentSentence
 from arekit.common.experiment.data_type import DataType
 from arekit.common.pipeline.base import BasePipelineLauncher
 from arekit.common.synonyms.grouping import SynonymsCollectionValuesGroupingProviders
@@ -33,8 +31,6 @@
 from arelight.synonyms import iter_synonym_groups
 from arelight.utils import IdAssigner, get_default_download_dir
 
-from ru_sent_tokenize import ru_sent_tokenize
-
 
 class TestInfer(unittest.TestCase):
 
@@ -57,17 +53,6 @@ def iter_groups(filepath):
             for data in iter_synonym_groups(file):
                 yield data
 
-    @staticmethod
-    def input_to_docs(texts):
-        assert(isinstance(texts, list))
-        docs = []
-        for doc_id, contents in enumerate(texts):
-            sentences = ru_sent_tokenize(contents)
-            sentences = list(map(lambda text: BaseDocumentSentence(text), sentences))
-            doc = Document(doc_id=doc_id, sentences=sentences)
-            docs.append(doc)
-        return docs
-
     def create_sampling_params(self):
 
         target_func = lambda data_type: join(utils.TEST_OUT_DIR, "-".join(["samples", data_type.name.lower()]))
@@ -111,7 +96,7 @@ def launch(self, pipeline):
             synonyms=synonyms,
             dist_in_terms_bound=100,
             dist_in_sentences=0,
-            doc_provider=utils.InMemoryDocProvider(docs=self.input_to_docs(actual_content)),
+            doc_provider=utils.InMemoryDocProvider(docs=utils.input_to_docs(actual_content)),
             terms_per_context=50,
             text_pipeline=text_parser)