Merge pull request #274 from didier-84/master

Fix sentence segmentation bug
twitter · Apr 28, 2024 · 4fcebf6 · 4fcebf6
2 parents 65d0a66 + d9730a4
commit 4fcebf6
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/lib/twitter_cldr/segmentation/rule_set.rb b/lib/twitter_cldr/segmentation/rule_set.rb
@@ -31,7 +31,7 @@ def each_boundary(cursor, stop = cursor.length)
 
         until cursor.position >= stop || cursor.eos?
           state_machine.handle_next(cursor)
-          yield cursor.position if suppressions.should_break?(cursor)
+          yield cursor.position if cursor.eos? || suppressions.should_break?(cursor)
         end
       end
 

diff --git a/spec/segmentation/break_iterator_spec.rb b/spec/segmentation/break_iterator_spec.rb
@@ -52,6 +52,13 @@
           "She's nice."
         ])
       end
+
+      it "splits correctly when a string ends with an exception directly followed by a single space" do
+        str = "I like the Mrs. "
+        expect(iterator.each_sentence(str).map { |word, _, _| word }).to eq([
+          "I like the Mrs. "
+        ])
+      end
     end
 
     context "without ULI exceptions" do