fix paragraphs split logic

WorksApplications · Dec 23, 2023 · c1a765d · c1a765d
1 parent 657e211
commit c1a765d
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/lib/src/main/scala/com/worksap/nlp/uzushio/lib/utils/Paragraphs.scala b/lib/src/main/scala/com/worksap/nlp/uzushio/lib/utils/Paragraphs.scala
@@ -11,7 +11,7 @@ object Paragraphs {
   final val FULLWIDTH_SPACE: Char = '　'
 
   def extractCleanParagraphs(text: String): Seq[String] = {
-    val paragraphs = StringUtils.split(text, PARAGRAPH_SEP)
+    val paragraphs = StringUtils.splitByWholeSeparator(text, PARAGRAPH_SEP)
     paragraphs.flatMap { x =>
       val par = extractCleanParagraph(x)
       if (hasContent(par)) Some(par) else None

diff --git a/lib/src/test/scala/com/worksap/nlp/uzushio/lib/utils/ParagraphsSpec.scala b/lib/src/test/scala/com/worksap/nlp/uzushio/lib/utils/ParagraphsSpec.scala
@@ -0,0 +1,12 @@
+package com.worksap.nlp.uzushio.lib.utils
+
+import org.scalatest.freespec.AnyFreeSpec
+
+class ParagraphsSpec extends AnyFreeSpec {
+  "correctly splits paragraphs" in {
+    val doc = "test1\n\ntest2\ntest3"
+    val pars = Paragraphs.extractCleanParagraphs(doc)
+    assert(pars.length == 2)
+    assert(pars == Seq("test1", "test2\ntest3"))
+  }
+}