Merge pull request #74 from nipunsadvilkar/npn-newline-fix

CHANGELOG.md

@@ -1,3 +1,13 @@
+# v0.3.1
+- 🚑 ✅ Handle Newline character & update tests
+
+# v0.3.0
+-   ✨ 💫  Support Multiple languages - \#2
+-   🏎⚡️💯 Benchmark across Segmentation Tools, Libraries and Algorithms
+-   🎨 ♻️ Update sentence char_span logic
+-   ⚡️  Performance improvements - \#41
+-   ♻️🐛 Refactor AbbreviationReplacer
+
 # v0.3.0rc
 -   ✨ 💫 sent `char_span` through with spaCy & regex approach - \#63
 -   ♻️ Refactoring to support multiple languages

pysbd/about.py

@@ -2,7 +2,7 @@
 # https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
 
 __title__ = "pysbd"
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 __summary__ = "pysbd (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
 __uri__ = "http://nipunsadvilkar.github.io/"
 __author__ = "Nipun Sadvilkar"

pysbd/processor.py

@@ -28,6 +28,7 @@ def __init__(self, text, lang, char_span=False):
     def process(self):
         if not self.text:
             return self.text
+        self.text = self.text.replace('\n', '\r')
         li = ListItemReplacer(self.text)
         self.text = li.add_line_break()
         self.replace_abbreviations()

tests/regression/test_issues.py

@@ -58,7 +58,13 @@
     ('#55', "She turned to him, \"This is great.\" She held the book out to show him.",
         [
             ('She turned to him, "This is great." ', 0, 36), ('She held the book out to show him.', 36, 70)
-        ])
+        ]),
+('#56',
+"""This eBook is for the use of anyone anywhere at no cost
+you may copy it, give it away or re-use it under the terms of the this license
+""",
+[('This eBook is for the use of anyone anywhere at no cost\n', 0, 56),
+ ('you may copy it, give it away or re-use it under the terms of the this license\n', 56, 135)])
 
 ]