Merge pull request #6 from valencik/more-analyzers

Add asciiFolderWithLower
cozydev-pink · Jun 22, 2022 · 17f27a6 · 17f27a6
2 parents 7b06df5 + e3fa6f8
commit 17f27a6
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 0 deletions.
diff --git a/lucene/src/main/scala/textmogrify/lucene/Analyzers.scala b/lucene/src/main/scala/textmogrify/lucene/Analyzers.scala
@@ -45,4 +45,13 @@ object Analyzers {
       }
     }
 
+  def asciiFolderWithLower(): Analyzer =
+    new Analyzer {
+      protected def createComponents(fieldName: String): TokenStreamComponents = {
+        val source = new StandardTokenizer()
+        val tokens = new LowerCaseFilter(source)
+        new TokenStreamComponents(source, new ASCIIFoldingFilter(tokens))
+      }
+    }
+
 }
diff --git a/lucene/src/test/scala/textmogrify/lucene/AnalyzersSuite.scala b/lucene/src/test/scala/textmogrify/lucene/AnalyzersSuite.scala
@@ -30,4 +30,12 @@ class AnalyzersSuite extends CatsEffectSuite {
     assertIO(actual, Vector("I", "like", "jalapenos"))
   }
 
+  test("asciiFolderLower should fold and lowercase") {
+    val tokenizer = AnalyzerResource.tokenizer[IO](Analyzers.asciiFolderWithLower())
+    val actual = tokenizer.use { f =>
+      f("I Like Jalapeños")
+    }
+    assertIO(actual, Vector("i", "like", "jalapenos"))
+  }
+
 }