Merge pull request #33 from valencik/more-langs

Add More Languages
cozydev-pink · Sep 13, 2022 · ff09cba · ff09cba
2 parents 70b594a + 3b2daa4
commit ff09cba
Show file tree

Hide file tree

Showing 2 changed files with 150 additions and 0 deletions.
diff --git a/lucene/src/main/scala/textmogrify/lucene/AnalyzerBuilder.scala b/lucene/src/main/scala/textmogrify/lucene/AnalyzerBuilder.scala
@@ -22,6 +22,8 @@ import org.apache.lucene.analysis.standard.StandardTokenizer
 import org.apache.lucene.analysis.en.PorterStemFilter
 import org.apache.lucene.analysis.es.SpanishLightStemFilter
 import org.apache.lucene.analysis.fr.FrenchLightStemFilter
+import org.apache.lucene.analysis.it.ItalianLightStemFilter
+import org.apache.lucene.analysis.de.GermanLightStemFilter
 import org.apache.lucene.analysis.LowerCaseFilter
 import org.apache.lucene.analysis.Analyzer
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter
@@ -103,6 +105,10 @@ object AnalyzerBuilder {
     new EnglishAnalyzerBuilder(Config.empty, false)
   def french: FrenchAnalyzerBuilder =
     new FrenchAnalyzerBuilder(Config.empty, false)
+  def german: GermanAnalyzerBuilder =
+    new GermanAnalyzerBuilder(Config.empty, false)
+  def italian: ItalianAnalyzerBuilder =
+    new ItalianAnalyzerBuilder(Config.empty, false)
   def spanish: SpanishAnalyzerBuilder =
     new SpanishAnalyzerBuilder(Config.empty, false)
 }
@@ -204,3 +210,55 @@ final class SpanishAnalyzerBuilder private[lucene] (
   def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
     mkFromStandardTokenizer(config)(ts => if (self.stemmer) new SpanishLightStemFilter(ts) else ts)
 }
+
+final class ItalianAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = ItalianAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): ItalianAnalyzerBuilder =
+    new ItalianAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): ItalianAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** Adds the ItalianLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required for the Lucene ItalianLightStemFilter.
+    */
+  def withItalianLightStemmer: ItalianAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config)(ts => if (self.stemmer) new ItalianLightStemFilter(ts) else ts)
+}
+
+final class GermanAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = GermanAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): GermanAnalyzerBuilder =
+    new GermanAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): GermanAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** Adds the GermanLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required for the Lucene GermanLightStemFilter.
+    */
+  def withGermanLightStemmer: GermanAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config)(ts => if (self.stemmer) new GermanLightStemFilter(ts) else ts)
+}
diff --git a/lucene/src/test/scala/textmogrify/lucene/AnalyzerBuilderSuite.scala b/lucene/src/test/scala/textmogrify/lucene/AnalyzerBuilderSuite.scala
@@ -189,3 +189,95 @@ class SpanishAnalyzerBuilderSuite extends CatsEffectSuite {
   }
 
 }
+
+class ItalianAnalyzerBuilderSuite extends CatsEffectSuite {
+
+  val jalapenos = "Mi piacciono i jalapeños"
+  val jumping = "A Neeko piace saltare sui contatori"
+
+  test("italian analyzer default should tokenize without any transformations") {
+    val analyzer = AnalyzerBuilder.italian
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Mi", "piacciono", "i", "jalapeños"))
+  }
+
+  test("italian analyzer withLowerCasing should lowercase all letters") {
+    val analyzer = AnalyzerBuilder.italian.withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("mi", "piacciono", "i", "jalapeños"))
+  }
+
+  test("italian analyzer withASCIIFolding should fold 'ñ' to 'n'") {
+    val analyzer = AnalyzerBuilder.italian.withASCIIFolding
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Mi", "piacciono", "i", "jalapenos"))
+  }
+
+  test("italian analyzer withStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.italian.withStopWords(Set("i"))
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Mi", "piacciono", "jalapeños"))
+  }
+
+  test("italian analyzer withItalianLightStemmer should lowercase and stem words") {
+    val analyzer = AnalyzerBuilder.italian.withItalianLightStemmer
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("a", "neeko", "piace", "saltar", "sui", "contator"))
+  }
+
+  test("italian analyzer builder settings can be chained") {
+    val analyzer = AnalyzerBuilder.italian.withItalianLightStemmer
+      .withStopWords(Set("a", "sui"))
+      .withASCIIFolding
+      .withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("neeko", "piace", "saltar", "contator"))
+  }
+
+}
+
+class GermanAnalyzerBuilderSuite extends CatsEffectSuite {
+
+  val jalapenos = "Ich mag Jalapeños"
+  val jumping = "Neeko springt gerne auf Theken"
+
+  test("german analyzer default should tokenize without any transformations") {
+    val analyzer = AnalyzerBuilder.german
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Ich", "mag", "Jalapeños"))
+  }
+
+  test("german analyzer withLowerCasing should lowercase all letters") {
+    val analyzer = AnalyzerBuilder.german.withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("ich", "mag", "jalapeños"))
+  }
+
+  test("german analyzer withASCIIFolding should fold 'ñ' to 'n'") {
+    val analyzer = AnalyzerBuilder.german.withASCIIFolding
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Ich", "mag", "Jalapenos"))
+  }
+
+  test("german analyzer withStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.german.withStopWords(Set("Ich"))
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("mag", "Jalapeños"))
+  }
+
+  test("german analyzer withGermanLightStemmer should lowercase and stem words") {
+    val analyzer = AnalyzerBuilder.german.withGermanLightStemmer
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("neeko", "springt", "gern", "auf", "thek"))
+  }
+
+  test("german analyzer builder settings can be chained") {
+    val analyzer = AnalyzerBuilder.german.withGermanLightStemmer
+      .withStopWords(Set("auf"))
+      .withASCIIFolding
+      .withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("neeko", "springt", "gern", "thek"))
+  }
+
+}