🚚 Renaming package & ♻️Refactor scripts (#26)

nipunsadvilkar · web-flow · commit ccb75b501a3a · 2019-10-09T23:08:55.000+05:30
🚚 Renaming package &amp; ♻️Refactor scripts
diff --git a/.gitignore b/.gitignore
@@ -45,6 +45,7 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
+.pytest_cache/
 
 # Translations
 *.mo
@@ -99,3 +100,6 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# vscode
+.vscode/
diff --git a/README.md b/README.md
@@ -1,42 +1,39 @@
 # pySBD: Python Sentence Boundary Disambiguation (SBD)
 
-[![Build Status](https://travis-ci.org/nipunsadvilkar/pySBD.svg?branch=master)](https://travis-ci.org/nipunsadvilkar/pySBD)
-[![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/nipunsadvilkar/pySBD/blob/master/LICENSE)
+[![Build Status](https://travis-ci.org/nipunsadvilkar/pySBD.svg?branch=master)](https://travis-ci.org/nipunsadvilkar/pySBD) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/nipunsadvilkar/pySBD/blob/master/LICENSE)
 
 pySBD - python Sentence Boundary Disambiguation (SBD) - is a rule-based sentence boundary detection module that works out-of-the-box.
 
 This project is a direct port of ruby gem - [Pragmatic Segmenter](https://github.com/diasks2/pragmatic_segmenter) which provides rule-based sentence boundary detection.
 
-
 ## Install
 
 **Python**
-```
-pip install pysbd
-```
+
+    pip install pysbd
 
 ## Usage
 
-* Currently pySBD support only English language. Support for more languages will be released soon.
+-   Currently pySBD supports only English language. Support for more languages will be released soon.
 
 ```python
-import pySBD
+import pysbd
 text = "Hello World. My name is Jonas."
-seg = pySBD.Segmenter(text, clean=False)
-print(seg.segment())
+seg = pysbd.Segmenter(language="en", clean=False)
+print(seg.segment(text))
 # ['Hello World.', 'My name is Jonas.']
 ```
 
 ## Contributing
 
 If you find a text that is incorrectly segmented using pySBD, please submit an issue.
 
-1. Fork it ( https://github.com/nipunsadvilkar/pySBD/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+1.  Fork it ( https://github.com/nipunsadvilkar/pySBD/fork )
+2.  Create your feature branch (`git checkout -b my-new-feature`)
+3.  Commit your changes (`git commit -am 'Add some feature'`)
+4.  Push to the branch (`git push origin my-new-feature`)
+5.  Create a new Pull Request
 
+## Credit
 
-# Credit
 This project wouldn't be possible without the great work done by [Pragmatic Segmenter](https://github.com/diasks2/pragmatic_segmenter) team.
diff --git a/pysbd/__init__.py b/pysbd/__init__.py
diff --git a/pysbd/abbreviation_replacer.py b/pysbd/abbreviation_replacer.py
@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 import re
-from pySBD.rules import Text
+from pysbd.rules import Text
 # TODO: SENTENCE_STARTERS should be lang specific
-from pySBD.lang.standard import Abbreviation, SENTENCE_STARTERS
-from pySBD.lang.common.numbers import (Common, SingleLetterAbbreviationRules,
+from pysbd.lang.standard import Abbreviation, SENTENCE_STARTERS
+from pysbd.lang.common.numbers import (Common, SingleLetterAbbreviationRules,
                                        AmPmRules)
 
 
diff --git a/pysbd/about.py b/pysbd/about.py
@@ -1,9 +1,9 @@
 # inspired from:
 # https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
 
-__title__ = "pySBD"
-__version__ = "0.0.1"
-__summary__ = "pySBD (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
+__title__ = "pysbd"
+__version__ = "0.1.1"
+__summary__ = "pysbd (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
 __uri__ = "http://nipunsadvilkar.github.io/"
 __author__ = "Nipun Sadvilkar"
 __email__ = "nipunsadvilkar@gmail.com"
diff --git a/pysbd/between_punctuation.py b/pysbd/between_punctuation.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import re
 from functools import partial
-from pySBD.punctuation_replacer import replace_punctuation
+from pysbd.punctuation_replacer import replace_punctuation
 
 
 class BetweenPunctuation(object):
diff --git a/pysbd/clean/__init__.py b/pysbd/clean/__init__.py
diff --git a/pysbd/clean/rules.py b/pysbd/clean/rules.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from pySBD.rules import Rule
+from pysbd.rules import Rule
 
 
 class CleanRules(object):
diff --git a/pysbd/cleaner.py b/pysbd/cleaner.py
@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 import re
-from pySBD.rules import Text
-from pySBD.clean.rules import PDF, HTML, CleanRules as cr
-from pySBD.lang.standard import Abbreviation
+from pysbd.rules import Text
+from pysbd.clean.rules import PDF, HTML, CleanRules as cr
+from pysbd.lang.standard import Abbreviation
 
 
 class Cleaner(object):
diff --git a/pysbd/exclamation_words.py b/pysbd/exclamation_words.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
-from pySBD.punctuation_replacer import replace_punctuation
+from pysbd.punctuation_replacer import replace_punctuation
 
 
 class ExclamationWords(object):
diff --git a/pysbd/lang/__init__.py b/pysbd/lang/__init__.py
diff --git a/pysbd/lang/common/__init__.py b/pysbd/lang/common/__init__.py
diff --git a/pysbd/lang/common/ellipsis.py b/pysbd/lang/common/ellipsis.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from pySBD.rules import Rule
+from pysbd.rules import Rule
 
 
 class EllipsisRules(object):
diff --git a/pysbd/lang/common/numbers.py b/pysbd/lang/common/numbers.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
-from pySBD.rules import Rule
+from pysbd.rules import Rule
 
 
 class Common(object):
diff --git a/pysbd/lang/english.py b/pysbd/lang/english.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-from pySBD.lang.common.numbers import Common
-from pySBD.cleaner import Cleaner
-# from pySBD.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common.numbers import Common
+from pysbd.cleaner import Cleaner
+# from pysbd.abbreviation_replacer import AbbreviationReplacer
 
 
 class English(Common):
diff --git a/pysbd/lang/standard.py b/pysbd/lang/standard.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from pySBD.rules import Rule
+from pysbd.rules import Rule
 
 
 class Standard(object):
diff --git a/pysbd/languages.py b/pysbd/languages.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-from pySBD.lang.standard import Standard
-from pySBD.lang.english import English
+from pysbd.lang.standard import Standard
+from pysbd.lang.english import English
 
 LANGUAGE_CODES = {'en': English}
 
diff --git a/pysbd/lists_item_replacer.py b/pysbd/lists_item_replacer.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import string
 import re
-from pySBD.rules import Rule, Text
+from pysbd.rules import Rule, Text
 from functools import partial
 
 
diff --git a/pysbd/processor.py b/pysbd/processor.py
@@ -1,18 +1,18 @@
 # -*- coding: utf-8 -*-
 import re
 import os
-from pySBD.rules import Text
-from pySBD.lists_item_replacer import ListItemReplacer
-from pySBD.languages import Language
-from pySBD.lang.standard import (Standard, Abbreviation,
+from pysbd.rules import Text
+from pysbd.lists_item_replacer import ListItemReplacer
+from pysbd.languages import Language
+from pysbd.lang.standard import (Standard, Abbreviation,
                                  DoublePunctuationRules,
                                  ExclamationPointRules, SubSymbolsRules,
                                  ReinsertEllipsisRules)
-from pySBD.lang.common.numbers import Common, Numbers
-from pySBD.lang.common.ellipsis import EllipsisRules
-from pySBD.exclamation_words import ExclamationWords
-from pySBD.between_punctuation import BetweenPunctuation
-from pySBD.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common.numbers import Common, Numbers
+from pysbd.lang.common.ellipsis import EllipsisRules
+from pysbd.exclamation_words import ExclamationWords
+from pysbd.between_punctuation import BetweenPunctuation
+from pysbd.abbreviation_replacer import AbbreviationReplacer
 
 
 class Processor(object):
@@ -30,50 +30,62 @@ def process(self):
         self.text = li.add_line_break()
         self.text = AbbreviationReplacer(self.text).replace()
         self.replace_numbers()
-        self.text = self.replace_continuous_punctuation()
-        self.text = self.replace_periods_before_numeric_references()
-        self.text = Text(self.text).apply(Abbreviation.WithMultiplePeriodsAndEmailRule)
-        self.text = Text(self.text).apply(Standard.GeoLocationRule)
-        self.text = Text(self.text).apply(Standard.FileFormatRule)
+        self.replace_continuous_punctuation()
+        self.replace_periods_before_numeric_references()
+        self.text = Text(self.text).apply(
+            Abbreviation.WithMultiplePeriodsAndEmailRule,
+            Standard.GeoLocationRule, Standard.FileFormatRule)
         processed = self.split_into_segments()
         return processed
 
+    def rm_none_flatten(self, sents):
+        """Remove None values and unpack list of list sents
+
+        Parameters
+        ----------
+        sents : list
+            list of sentences
+
+        Returns
+        -------
+        list
+            unpacked and None removed list of sents
+        """
+        sents = list(filter(None, sents))
+        if not any(isinstance(s, list) for s in sents):
+            return sents
+        new_sents = []
+        for sent in sents:
+            if isinstance(sent, list):
+                for s in sent:
+                    new_sents.append(s)
+            else:
+                new_sents.append(sent)
+        return new_sents
+
     def split_into_segments(self):
         self.check_for_parens_between_quotes()
         sents = self.text.split('\r')
         # remove empty and none values
-        # https://stackoverflow.com/questions/3845423/remove-empty-strings-from-a-list-of-strings
-        sents = list(filter(None, sents))
-        # https://stackoverflow.com/questions/4698493/can-i-add-custom-methods-attributes-to-built-in-python-types
+        sents = self.rm_none_flatten(sents)
         sents = [
             Text(s).apply(Standard.SingleNewLineRule, *EllipsisRules.All)
             for s in sents
         ]
-        new_sents = [self.check_for_punctuation(s) for s in sents]
+        sents = [self.check_for_punctuation(s) for s in sents]
         # flatten list of list of sentences
-        if any(isinstance(s, list) for s in new_sents):
-            new_sents = [s for sents in new_sents for s in sents]
+        sents = self.rm_none_flatten(sents)
         sents = [
             Text(s).apply(*SubSymbolsRules.All)
-            for s in new_sents
+            for s in sents
         ]
         post_process_sents = [self.post_process_segments(s) for s in sents]
-        # TODO: Refactor to flatten and remove None and empty lists
-        # remove any empty or null values
-        sents = []
-        for s in post_process_sents:
-            if not s:
-                continue
-            if isinstance(s, list):
-                for l in s:
-                    sents.append(l)
-            else:
-                sents.append(s)
-        sents = [
+        post_process_sents = self.rm_none_flatten(post_process_sents)
+        post_process_sents = [
             Text(s).apply(Standard.SubSingleQuoteRule)
-            for s in sents
+            for s in post_process_sents
         ]
-        return sents
+        return post_process_sents
 
     def post_process_segments(self, txt):
         if len(txt) > 2 and re.search(r'\A[a-zA-Z]*\Z', txt):
@@ -105,12 +117,12 @@ def continuous_puncs_replace(match):
             sub1 = re.sub(re.escape('!'), '&ᓴ&', match)
             sub2 = re.sub(re.escape('?'), '&ᓷ&', sub1)
             return sub2
-        return re.sub(Common.CONTINUOUS_PUNCTUATION_REGEX,
+        self.text = re.sub(Common.CONTINUOUS_PUNCTUATION_REGEX,
                         continuous_puncs_replace, self.text)
 
     def replace_periods_before_numeric_references(self):
         # https://github.com/diasks2/pragmatic_segmenter/commit/d9ec1a352aff92b91e2e572c30bb9561eb42c703
-        return re.sub(Common.NUMBERED_REFERENCE_REGEX,
+        self.text = re.sub(Common.NUMBERED_REFERENCE_REGEX,
                       r"∯\2\r\7", self.text)
 
     def consecutive_underscore(self, txt):
diff --git a/pysbd/punctuation_replacer.py b/pysbd/punctuation_replacer.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
-from pySBD.rules import Rule, Text
+from pysbd.rules import Rule, Text
 
 
 class EscapeRegexReservedCharacters(object):
diff --git a/pysbd/rules.py b/pysbd/rules.py
@@ -14,23 +14,32 @@ def __init__(self, pattern, replacement):
         self.replacement = replacement
 
     def __repr__(self):
-        return '<{} pattern={} and replacement={}>'.format(
+        return '<{} pattern="{}" and replacement="{}">'.format(
             self.__class__.__name__, self.pattern, self.replacement)
 
 
 class Text(str):
+    """Extending str functionality to apply regex rules
 
+    https://stackoverflow.com/questions/4698493/can-i-add-custom-methods-attributes-to-built-in-python-types
+
+    Parameters
+    ----------
+    str : str
+        string content
+
+    Returns
+    -------
+    str
+        input as it is if rule pattern doesnt match
+        else replacing found pattern with replacement chars
+    """
     def apply(self, *rules):
         for each_r in rules:
             self = re.sub(each_r.pattern, each_r.replacement, self)
             # print(self, each_r)
         return self
 
-
-def sample_segment(s, expected_s):
-    return expected_s
-
-
 if __name__ == "__main__":
     SubstituteListPeriodRule = Rule('♨', '∯')
     StdRule = Rule(r'∯', r'∯♨')
diff --git a/pysbd/segmenter.py b/pysbd/segmenter.py
diff --git a/setup.py b/setup.py
diff --git a/tests/test_english.py b/tests/test_english.py
diff --git a/tests/test_english_clean.py b/tests/test_english_clean.py
