Merge branch 'master' into mn-benchmark

Author: nipunsadvilkar

pysbd/exclamation_words.py

@@ -15,4 +15,3 @@ class ExclamationWords(object):
     def apply_rules(cls, text):
         return re.sub(ExclamationWords.EXCLAMATION_REGEX, replace_punctuation,
                       text)
-

pysbd/processor.py

@@ -1,15 +1,11 @@
 # -*- coding: utf-8 -*-
 import re
-import spacy
 from pysbd.utils import Text, TextSpan
 from pysbd.lists_item_replacer import ListItemReplacer
 from pysbd.exclamation_words import ExclamationWords
 from pysbd.between_punctuation import BetweenPunctuation
 from pysbd.abbreviation_replacer import AbbreviationReplacer
 
-nlp = spacy.blank('en')
-
-
 class Processor(object):
 
     def __init__(self, text, lang, char_span=False):
@@ -28,7 +24,6 @@ def __init__(self, text, lang, char_span=False):
         self.text = text
         self.lang = lang
         self.char_span = char_span
-        self.doc = nlp.make_doc(self.text)
 
     def process(self):
         if not self.text:
@@ -42,20 +37,8 @@ def process(self):
         self.text = Text(self.text).apply(
             self.lang.Abbreviation.WithMultiplePeriodsAndEmailRule,
             self.lang.GeoLocationRule, self.lang.FileFormatRule)
-        processed = self.split_into_segments()
-        if self.char_span:
-            return self.sentences_with_char_spans(processed)
-        else:
-            return processed
-
-    def sentences_with_char_spans(self, sentences):
-        sent_start_token_idx = [m.start() for sent in sentences for m in re.finditer(re.escape(sent), self.doc.text)]
-        for tok in self.doc:
-            if tok.idx in sent_start_token_idx:
-                tok.is_sent_start = True
-            else:
-                tok.is_sent_start = False
-        return [TextSpan(sent.text_with_ws, sent.start_char, sent.end_char) for sent in self.doc.sents]
+        postprocessed_sents = self.split_into_segments()
+        return postprocessed_sents
 
     def rm_none_flatten(self, sents):
         """Remove None values and unpack list of list sents

pysbd/segmenter.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
+import re
+
 from pysbd.languages import Language
 from pysbd.processor import Processor
 from pysbd.cleaner import Cleaner
+from pysbd.utils import TextSpan
 
 class Segmenter(object):
 
@@ -44,16 +47,30 @@ def processor(self, text):
             return Processor(text, self.language_module,
                              char_span=self.char_span)
 
+    def sentences_with_char_spans(self, sentences):
+        # since SENTENCE_BOUNDARY_REGEX doesnt account
+        # for trailing whitespaces \s* is used as suffix
+        # to keep non-destructive text after segments joins
+        return [TextSpan(m.group(), m.start(), m.end()) for sent in sentences
+                for m in re.finditer('{0}\s*'.format(re.escape(sent)),
+                self.original_text)]
+
     def segment(self, text):
+        self.original_text = text
         if not text:
             return []
         if self.clean and self.char_span:
             raise ValueError("char_span must be False if clean is True. "
                              "Since `clean=True` will modify original text.")
-        if self.language != 'en' and self.char_span:
-            raise ValueError("char_span functionality not supported for "
-                             "languages other than English (`en`)")
         elif self.clean:
             text = self.cleaner(text).clean()
-        segments = self.processor(text).process()
-        return segments
+        postprocessed_sents = self.processor(text).process()
+        sentence_w_char_spans = self.sentences_with_char_spans(postprocessed_sents)
+        if self.clean:
+            # clean and destructed sentences
+            return postprocessed_sents
+        elif self.char_span:
+            return sentence_w_char_spans
+        else:
+            # nondestructive with whitespaces
+            return [textspan.sent for textspan in sentence_w_char_spans]

tests/lang/test_amharic.py

@@ -10,4 +10,5 @@
 def test_am_sbd(am_default_fixture, text, expected_sents):
     """Amharic language SBD tests"""
     segments = am_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents

tests/lang/test_arabic.py

@@ -20,4 +20,5 @@
 def test_ar_sbd(ar_default_fixture, text, expected_sents):
     """Arabic language SBD tests"""
     segments = ar_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents

tests/lang/test_armenian.py

@@ -83,10 +83,12 @@
 def test_hy_sbd(hy_default_fixture, text, expected_sents):
     """Armenian language SBD tests"""
     segments = hy_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents
 
 @pytest.mark.parametrize('text,expected_sents', HY_MORE_TEST_CASES)
 def test_hy_sbd_more(hy_default_fixture, text, expected_sents):
     """Armenian language SBD tests"""
     segments = hy_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents

tests/lang/test_bulgarian.py

@@ -16,4 +16,5 @@
 def test_bg_sbd(bg_default_fixture, text, expected_sents):
     """Bulgarian language SBD tests"""
     segments = bg_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents

tests/lang/test_burmese.py

@@ -10,4 +10,5 @@
 def test_my_sbd(my_default_fixture, text, expected_sents):
     """Burmese language SBD tests"""
     segments = my_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents

tests/lang/test_chinese.py

@@ -12,4 +12,5 @@
 def test_zsh_sbd(zh_default_fixture, text, expected_sents):
     """Chinese language SBD tests from Pragmatic Segmenter"""
     segments = zh_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents

tests/lang/test_danish.py

@@ -101,6 +101,7 @@
 def test_da_sbd(da_default_fixture, text, expected_sents):
     """Danish language SBD tests"""
     segments = da_default_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents
 
 DA_RULES_CLEAN_TEST_CASES = [
@@ -117,11 +118,13 @@ def test_da_sbd(da_default_fixture, text, expected_sents):
 def test_da_sbd_clean(da_with_clean_no_span_fixture, text, expected_sents):
     """Danish language SBD tests with text clean"""
     segments = da_with_clean_no_span_fixture.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents
 
 @pytest.mark.parametrize('text,expected_sents', DA_PDF_TEST_DATA)
 def test_da_pdf_type(text, expected_sents):
     """SBD tests from Pragmatic Segmenter for doctype:pdf"""
     seg = pysbd.Segmenter(language="da", clean=True, doc_type='pdf')
     segments = seg.segment(text)
+    segments = [s.strip() for s in segments]
     assert segments == expected_sents