Simplify diacratic removal; modify Latin & Greek preprocessors

ext/js/language/language-descriptors.js

@@ -19,9 +19,8 @@ import {removeArabicScriptDiacritics} from './ar/arabic-text-preprocessors.js';
 import {eszettPreprocessor} from './de/german-text-preprocessors.js';
 import {collapseEmphaticSequences, convertAlphabeticCharacters, convertHalfWidthCharacters, convertHiraganaToKatakana, convertNumericCharacters} from './ja/japanese-text-preprocessors.js';
 import {isStringPartiallyJapanese} from './ja/japanese.js';
-import {removeLatinDiacritics} from './la/latin-text-preprocessors.js';
 import {removeRussianDiacritics, yoToE} from './ru/russian-text-preprocessors.js';
-import {capitalizeFirstLetter, decapitalize} from './text-preprocessors.js';
+import {capitalizeFirstLetter, decapitalize, removeAlphabeticDiacritics} from './text-preprocessors.js';
 
 const capitalizationPreprocessors = {
     decapitalize,
@@ -84,7 +83,10 @@ const languageDescriptors = [
         iso: 'grc',
         name: 'Ancient Greek',
         exampleText: 'γράφω',
-        textPreprocessors: capitalizationPreprocessors
+        textPreprocessors: {
+            ...capitalizationPreprocessors,
+            removeAlphabeticDiacritics
+        }
     },
     {
         iso: 'hu',
@@ -109,7 +111,8 @@ const languageDescriptors = [
         name: 'Latin',
         exampleText: 'legere',
         textPreprocessors: {
-            removeLatinDiacritics
+            ...capitalizationPreprocessors,
+            removeAlphabeticDiacritics
         }
     },
     {

ext/js/language/text-preprocessors.js

@@ -33,3 +33,17 @@
     options: basicTextPreprocessorOptions,
     process: (str, setting) => (setting ? str.charAt(0).toUpperCase() + str.slice(1) : str)
 };
+
+/**
+ * WARNING: This should NOT be used with languages that use Han characters,
+ *          as it can result in undesirable normalization: 
+ *            - '\u9038'.normalize('NFD') => '\u9038' (逸) 
+ *            - '\ufa67'.normalize('NFD') => '\u9038' (逸 => 逸) 
+ * @type {import('language').TextPreprocessor<boolean>}
+ */
+export const removeAlphabeticDiacritics = {
+    name: 'Remove Alphabetic Diacritics',
+    description: 'ἄήé -> αηe',
+    options: basicTextPreprocessorOptions,
+    process: (str, setting) => (setting ? str.normalize('NFD').replace(/[\u0300-\u036f]/g, '') : str)
+};