fix(internet): userName, email and slugify return only ascii

src/modules/helpers/index.ts

@@ -33,8 +33,10 @@ export class HelpersModule {
    */
   slugify(string: string = ''): string {
     return string
-      .replace(/ /g, '-')
-      .replace(/[^\一-龠\ぁ-ゔ\ァ-ヴー\w\.\-]+/g, '');
+      .normalize('NFKD') //for example è decomposes to as e +  ̀
+      .replace(/[\u0300-\u036f]/g, '') // removes combining marks
+      .replace(/ /g, '-') // replaces spaces with hyphens
+      .replace(/[^\w\.\-]+/g, ''); // removes all non-word characters except for dots and hyphens
   }
 
   /**

src/modules/internet/index.ts

@@ -81,10 +81,7 @@ export class InternetModule {
         this.faker.definitions.internet.free_email
       );
 
-    let localPart: string = this.faker.helpers.slugify(
-      this.userName(firstName, lastName)
-    );
-
+    let localPart: string = this.userName(firstName, lastName);
     if (options?.allowSpecialCharacters) {
       const usernameChars: string[] = '._-'.split('');
       const specialChars: string[] = ".!#$%&'*+-/=?^_`{|}~".split('');
@@ -125,7 +122,7 @@ export class InternetModule {
   }
 
   /**
-   * Generates a username using the given person's name as base.
+   * Generates a username using the given person's name as base. This will always return a plain ASCII string.
    *
    * @param firstName The optional first name to use. If not specified, a random one will be chosen.
    * @param lastName The optional last name to use. If not specified, a random one will be chosen.
@@ -155,8 +152,102 @@ export class InternetModule {
         ])}${lastName}${this.faker.datatype.number(99)}`;
         break;
     }
+
+    //There may still be non-ascii characters in the result.
+    //First remove simple accents etc
+    result = result
+      .normalize('NFKD') //for example è decomposes to as e +  ̀
+      .replace(/[\u0300-\u036f]/g, ''); // removes combining marks
+
+    //simple mapping for Cyrillic - FIXME we could also do this for some other simple alphabets like Greek and Thai
+    const mappings: { [key: string]: string } = {
+      Ё: 'YO',
+      Й: 'I',
+      Ц: 'TS',
+      У: 'U',
+      К: 'K',
+      Е: 'E',
+      Н: 'N',
+      Г: 'G',
+      Ш: 'SH',
+      Щ: 'SCH',
+      З: 'Z',
+      Х: 'H',
+      Ъ: "'",
+      ё: 'yo',
+      й: 'i',
+      ц: 'ts',
+      у: 'u',
+      к: 'k',
+      е: 'e',
+      н: 'n',
+      г: 'g',
+      ш: 'sh',
+      щ: 'sch',
+      з: 'z',
+      х: 'h',
+      ъ: "'",
+      Ф: 'F',
+      Ы: 'I',
+      В: 'V',
+      А: 'А',
+      П: 'P',
+      Р: 'R',
+      О: 'O',
+      Л: 'L',
+      Д: 'D',
+      Ж: 'ZH',
+      Э: 'E',
+      ф: 'f',
+      ы: 'i',
+      в: 'v',
+      а: 'a',
+      п: 'p',
+      р: 'r',
+      о: 'o',
+      л: 'l',
+      д: 'd',
+      ж: 'zh',
+      э: 'e',
+      Я: 'Ya',
+      Ч: 'CH',
+      С: 'S',
+      М: 'M',
+      И: 'I',
+      Т: 'T',
+      Ь: "'",
+      Б: 'B',
+      Ю: 'YU',
+      я: 'ya',
+      ч: 'ch',
+      с: 's',
+      м: 'm',
+      и: 'i',
+      т: 't',
+      ь: "'",
+      б: 'b',
+      ю: 'yu',
+    };
+    result = result
+      .split('')
+      .map(function (char) {
+        //if we have a mapping for this character, use it
+        if (mappings[char]) {
+          return mappings[char];
+        }
+        if (char.charCodeAt(0) < 0x80) {
+          //keep ascii characters
+          return char;
+        }
+        //return the hex value for Chinese, Japanese, Korean etc
+        return char.charCodeAt(0).toString(16);
+      })
+      .join('');
+
+    //remove spaces and '
     result = result.toString().replace(/'/g, '');
     result = result.replace(/ /g, '');
+
     return result;
   }
 

test/helpers.spec.ts

@@ -176,9 +176,21 @@ describe('helpers', () => {
       });
 
       describe('slugify()', () => {
-        it('removes unwanted characters from URI string', () => {
-          expect(faker.helpers.slugify('Aiden.Harªann')).toBe('Aiden.Harann');
+        it('removes non-word characters from strings except . and -', () => {
+          expect(faker.helpers.slugify('foo bar')).toBe('foo-bar');
+          expect(faker.helpers.slugify('Faker is cool')).toBe('Faker-is-cool');
+          expect(faker.helpers.slugify('super*star')).toBe('superstar');
           expect(faker.helpers.slugify("d'angelo.net")).toBe('dangelo.net');
+          expect(faker.helpers.slugify('hello你好')).toBe('hello');
+        });
+      });
+
+      describe('slugify()', () => {
+        it('strips simple diacritics from strings', () => {
+          expect(faker.helpers.slugify('Aiden.Harªann')).toBe('Aiden.Haraann');
+          expect(faker.helpers.slugify('Adèle.Argüello')).toBe(
+            'Adele.Arguello'
+          );
         });
       });
 

test/internet.spec.ts

@@ -98,6 +98,20 @@ describe('internet', () => {
           expect(faker.definitions.internet.free_email).toContain(suffix);
         });
 
+        it('should return a valid email in every locale', () => {
+          for (const locale of Object.keys(faker.locales)) {
+            faker.setLocale(locale);
+            const email = faker.internet.email();
+
+            expect(email).toBeTruthy();
+            expect(email).toBeTypeOf('string');
+            expect(email).toSatisfy(
+              validator.isEmail,
+              `locale: ${locale} has invalid email: ${email}`
+            );
+          }
+        });
+
         it('should return an email with given firstName', () => {
           const email = faker.internet.email('Aiden.Harann55');
 
@@ -130,19 +144,6 @@ describe('internet', () => {
           expect(faker.definitions.internet.free_email).toContain(suffix);
         });
 
-        it('should return an email with japanese characters', () => {
-          const email = faker.internet.email('思源_唐3');
-
-          expect(email).toBeTruthy();
-          expect(email).toBeTypeOf('string');
-          expect(email).toSatisfy(validator.isEmail);
-
-          const [prefix, suffix] = email.split('@');
-
-          expect(prefix).toMatch(/^思源_唐3/);
-          expect(faker.definitions.internet.free_email).toContain(suffix);
-        });
-
         it('should return an email with special characters', () => {
           const email = faker.internet.email('Mike', 'Smith', null, {
             allowSpecialCharacters: true,
@@ -203,20 +204,6 @@ describe('internet', () => {
           expect(prefix).toMatch(/^Aiden([._]Harann)?\d*/);
         });
 
-        it('should return an email with the example suffix and japanese characters', () => {
-          const email = faker.internet.exampleEmail('思源_唐3');
-
-          expect(email).toBeTruthy();
-          expect(email).toBeTypeOf('string');
-          expect(email).toSatisfy(validator.isEmail);
-
-          const [prefix, suffix] = email.split('@');
-
-          expect(suffix).toMatch(/^example\.(com|net|org)$/);
-          expect(faker.definitions.internet.example_email).toContain(suffix);
-          expect(prefix).toMatch(/^思源_唐3/);
-        });
-
         it('should return an email with special characters', () => {
           const email = faker.internet.exampleEmail('Mike', 'Smith', {
             allowSpecialCharacters: true,