Update Emojibase and Twemoji and switch to IamCal (Slack-style) shortcodes

package.json

@@ -64,8 +64,8 @@
     "counterpart": "^0.18.6",
     "diff-dom": "^4.2.2",
     "diff-match-patch": "^1.0.5",
-    "emojibase-data": "^5.1.1",
-    "emojibase-regex": "^4.1.1",
+    "emojibase-data": "^6.2.0",
+    "emojibase-regex": "^5.1.3",
     "escape-html": "^1.0.3",
     "file-saver": "^2.0.5",
     "filesize": "6.1.0",

src/HtmlUtils.tsx

@@ -34,7 +34,7 @@ import { IExtendedSanitizeOptions } from './@types/sanitize-html';
 import linkifyMatrix from './linkify-matrix';
 import SettingsStore from './settings/SettingsStore';
 import { tryTransformPermalinkToLocalHref } from "./utils/permalinks/Permalinks";
-import { SHORTCODE_TO_EMOJI, getEmojiFromUnicode } from "./emoji";
+import { getEmojiFromUnicode, getShortcodes } from "./emoji";
 import ReplyThread from "./components/views/elements/ReplyThread";
 import { mediaFromMxc } from "./customisations/Media";
 
@@ -78,20 +78,8 @@ function mightContainEmoji(str: string): boolean {
  * @return {String} The shortcode (such as :thumbup:)
  */
 export function unicodeToShortcode(char: string): string {
-    const data = getEmojiFromUnicode(char);
-    return (data && data.shortcodes ? `:${data.shortcodes[0]}:` : '');
-}
-
-/**
- * Returns the unicode character for an emoji shortcode
- *
- * @param {String} shortcode The shortcode (such as :thumbup:)
- * @return {String} The emoji character; null if none exists
- */
-export function shortcodeToUnicode(shortcode: string): string {
-    shortcode = shortcode.slice(1, shortcode.length - 1);
-    const data = SHORTCODE_TO_EMOJI.get(shortcode);
-    return data ? data.unicode : null;
+    const shortcodes = getShortcodes(getEmojiFromUnicode(char));
+    return shortcodes.length > 0 ? `:${shortcodes[0]}:` : '';
 }
 
 export function processHtmlForSending(html: string): string {

src/autocomplete/EmojiProvider.tsx

@@ -25,8 +25,7 @@ import { PillCompletion } from './Components';
 import { ICompletion, ISelectionRange } from './Autocompleter';
 import { uniq, sortBy } from 'lodash';
 import SettingsStore from "../settings/SettingsStore";
-import { shortcodeToUnicode } from '../HtmlUtils';
-import { EMOJI, IEmoji } from '../emoji';
+import { EMOJI, IEmoji, getShortcodes } from '../emoji';
 
 import EMOTICON_REGEX from 'emojibase-regex/emoticon';
 
@@ -38,21 +37,26 @@ const EMOJI_REGEX = new RegExp('(' + EMOTICON_REGEX.source + '|(?:^|\\s):[+-\\w]
 
 interface IEmojiShort {
     emoji: IEmoji;
-    shortname: string;
+    shortcode: string;
+    altShortcodes: string[];
     _orderBy: number;
 }
 
-const EMOJI_SHORTNAMES: IEmojiShort[] = EMOJI.sort((a, b) => {
+const EMOJI_SHORTCODES: IEmojiShort[] = EMOJI.sort((a, b) => {
     if (a.group === b.group) {
         return a.order - b.order;
     }
     return a.group - b.group;
-}).map((emoji, index) => ({
-    emoji,
-    shortname: `:${emoji.shortcodes[0]}:`,
-    // Include the index so that we can preserve the original order
-    _orderBy: index,
-}));
+}).map((emoji, index) => {
+    const [shortcode, ...altShortcodes] = getShortcodes(emoji);
+    return {
+        emoji,
+        shortcode: shortcode ? `:${shortcode}:` : undefined,
+        altShortcodes: altShortcodes.map(s => `:${s}:`),
+        // Include the index so that we can preserve the original order
+        _orderBy: index,
+    };
+}).filter(emoji => emoji.shortcode);
 
 function score(query, space) {
     const index = space.indexOf(query);
@@ -69,15 +73,15 @@ export default class EmojiProvider extends AutocompleteProvider {
 
     constructor() {
         super(EMOJI_REGEX);
-        this.matcher = new QueryMatcher<IEmojiShort>(EMOJI_SHORTNAMES, {
-            keys: ['emoji.emoticon', 'shortname'],
+        this.matcher = new QueryMatcher<IEmojiShort>(EMOJI_SHORTCODES, {
+            keys: ['emoji.emoticon', 'shortcode'],
             funcs: [
-                (o) => o.emoji.shortcodes.length > 1 ? o.emoji.shortcodes.slice(1).map(s => `:${s}:`).join(" ") : "", // aliases
+                o => o.altShortcodes.join(" "), // aliases
             ],
             // For matching against ascii equivalents
             shouldMatchWordsOnly: false,
         });
-        this.nameMatcher = new QueryMatcher(EMOJI_SHORTNAMES, {
+        this.nameMatcher = new QueryMatcher(EMOJI_SHORTCODES, {
             keys: ['emoji.annotation'],
             // For removing punctuation
             shouldMatchWordsOnly: true,
@@ -105,34 +109,33 @@ export default class EmojiProvider extends AutocompleteProvider {
 
             const sorters = [];
             // make sure that emoticons come first
-            sorters.push((c) => score(matchedString, c.emoji.emoticon || ""));
+            sorters.push(c => score(matchedString, c.emoji.emoticon || ""));
 
-            // then sort by score (Infinity if matchedString not in shortname)
-            sorters.push((c) => score(matchedString, c.shortname));
+            // then sort by score (Infinity if matchedString not in shortcode)
+            sorters.push(c => score(matchedString, c.shortcode));
             // then sort by max score of all shortcodes, trim off the `:`
-            sorters.push((c) => Math.min(...c.emoji.shortcodes.map(s => score(matchedString.substring(1), s))));
-            // If the matchedString is not empty, sort by length of shortname. Example:
+            sorters.push(c => Math.min(
+                ...[c.shortcode, ...c.altShortcodes].map(s => score(matchedString.substring(1), s)),
+            ));
+            // If the matchedString is not empty, sort by length of shortcode. Example:
             //  matchedString = ":bookmark"
             //  completions = [":bookmark:", ":bookmark_tabs:", ...]
             if (matchedString.length > 1) {
-                sorters.push((c) => c.shortname.length);
+                sorters.push(c => c.shortcode.length);
             }
             // Finally, sort by original ordering
-            sorters.push((c) => c._orderBy);
+            sorters.push(c => c._orderBy);
             completions = sortBy(uniq(completions), sorters);
 
-            completions = completions.map(({ shortname }) => {
-                const unicode = shortcodeToUnicode(shortname);
-                return {
-                    completion: unicode,
-                    component: (
-                        <PillCompletion title={shortname} aria-label={unicode}>
-                            <span>{ unicode }</span>
-                        </PillCompletion>
-                    ),
-                    range,
-                };
-            }).slice(0, LIMIT);
+            completions = completions.map(c => ({
+                completion: c.emoji.unicode,
+                component: (
+                    <PillCompletion title={c.shortcode} aria-label={c.emoji.unicode}>
+                        <span>{ c.emoji.unicode }</span>
+                    </PillCompletion>
+                ),
+                range,
+            })).slice(0, LIMIT);
         }
         return completions;
     }

src/components/views/emojipicker/Preview.tsx

@@ -17,7 +17,7 @@ limitations under the License.
 
 import React from 'react';
 
-import { IEmoji } from "../../../emoji";
+import { IEmoji, getShortcodes } from "../../../emoji";
 import { replaceableComponent } from "../../../utils/replaceableComponent";
 
 interface IProps {
@@ -30,8 +30,8 @@ class Preview extends React.PureComponent<IProps> {
         const {
             unicode = "",
             annotation = "",
-            shortcodes: [shortcode = ""],
-        } = this.props.emoji || {};
+        } = this.props.emoji;
+        const shortcode = getShortcodes(this.props.emoji)[0];
 
         return (
             <div className="mx_EmojiPicker_footer mx_EmojiPicker_preview">
@@ -42,9 +42,9 @@ class Preview extends React.PureComponent<IProps> {
                     <div className="mx_EmojiPicker_name mx_EmojiPicker_preview_name">
                         {annotation}
                     </div>
-                    <div className="mx_EmojiPicker_shortcode">
-                        {shortcode}
-                    </div>
+                    { shortcode ?
+                        <div className="mx_EmojiPicker_shortcode">{shortcode}</div> :
+                        null }
                 </div>
             </div>
         );

src/components/views/emojipicker/QuickReactions.tsx

@@ -18,7 +18,7 @@ limitations under the License.
 import React from 'react';
 
 import { _t } from '../../../languageHandler';
-import { getEmojiFromUnicode, IEmoji } from "../../../emoji";
+import { getEmojiFromUnicode, getShortcodes, IEmoji } from "../../../emoji";
 import Emoji from "./Emoji";
 import { replaceableComponent } from "../../../utils/replaceableComponent";
 
@@ -62,14 +62,17 @@ class QuickReactions extends React.Component<IProps, IState> {
     };
 
     render() {
+        const shortcode = this.state.hover ? getShortcodes(this.state.hover)[0] : undefined;
         return (
             <section className="mx_EmojiPicker_footer mx_EmojiPicker_quick mx_EmojiPicker_category">
                 <h2 className="mx_EmojiPicker_quick_header mx_EmojiPicker_category_label">
                     {!this.state.hover
                         ? _t("Quick Reactions")
                         : <React.Fragment>
                             <span className="mx_EmojiPicker_name">{this.state.hover.annotation}</span>
-                            <span className="mx_EmojiPicker_shortcode">{this.state.hover.shortcodes[0]}</span>
+                            { shortcode ?
+                                <span className="mx_EmojiPicker_shortcode">{shortcode}</span> :
+                                null }
                         </React.Fragment>
                     }
                 </h2>

src/emoji.ts

@@ -15,14 +15,14 @@ limitations under the License.
 */
 
 import EMOJIBASE from 'emojibase-data/en/compact.json';
+import SHORTCODES from 'emojibase-data/en/shortcodes/iamcal.json';
 
 export interface IEmoji {
     annotation: string;
-    group: number;
+    group?: number;
     hexcode: string;
-    order: number;
-    shortcodes: string[];
-    tags: string[];
+    order?: number;
+    tags?: string[];
     unicode: string;
     emoticon?: string;
 }
@@ -34,10 +34,14 @@ interface IEmojiWithFilterString extends IEmoji {
 // The unicode is stored without the variant selector
 const UNICODE_TO_EMOJI = new Map<string, IEmojiWithFilterString>(); // not exported as gets for it are handled by getEmojiFromUnicode
 export const EMOTICON_TO_EMOJI = new Map<string, IEmojiWithFilterString>();
-export const SHORTCODE_TO_EMOJI = new Map<string, IEmojiWithFilterString>();
 
 export const getEmojiFromUnicode = unicode => UNICODE_TO_EMOJI.get(stripVariation(unicode));
 
+const toArray = (shortcodes?: string | string[]): string[] =>
+    typeof shortcodes === "string" ? [shortcodes] : (shortcodes ?? []);
+export const getShortcodes = (emoji: IEmoji): string[] =>
+    toArray(SHORTCODES[emoji.hexcode]);
+
 const EMOJIBASE_GROUP_ID_TO_CATEGORY = [
     "people", // smileys
     "people", // actually people
@@ -66,12 +70,14 @@ const ZERO_WIDTH_JOINER = "\u200D";
 
 // Store various mappings from unicode/emoticon/shortcode to the Emoji objects
 EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => {
+    const shortcodes = getShortcodes(emoji);
     const categoryId = EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group];
     if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) {
         DATA_BY_CATEGORY[categoryId].push(emoji);
     }
+
     // This is used as the string to match the query against when filtering emojis
-    emoji.filterString = (`${emoji.annotation}\n${emoji.shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` +
+    emoji.filterString = (`${emoji.annotation}\n${shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` +
         `${emoji.unicode.split(ZERO_WIDTH_JOINER).join("\n")}`).toLowerCase();
 
     // Add mapping from unicode to Emoji object
@@ -87,13 +93,6 @@ EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => {
         // Add mapping from emoticon to Emoji object
         EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji);
     }
-
-    if (emoji.shortcodes) {
-        // Add mapping from each shortcode to Emoji object
-        emoji.shortcodes.forEach(shortcode => {
-            SHORTCODE_TO_EMOJI.set(shortcode, emoji);
-        });
-    }
 });
 
 /**

yarn.lock

@@ -3022,15 +3022,15 @@ emoji-regex@^8.0.0:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
   integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
 
-emojibase-data@^5.1.1:
-  version "5.1.1"
-  resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-5.1.1.tgz#0a0d63dd07ce1376b3d27642f28cafa46f651de6"
-  integrity sha512-za/ma5SfogHjwUmGFnDbTvSfm8GGFvFaPS27GPti16YZSp5EPgz+UDsZCATXvJGit+oRNBbG/FtybXHKi2UQgQ==
+emojibase-data@^6.2.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-6.2.0.tgz#db6c75c36905284fa623f4aa5f468d2be6ed364a"
+  integrity sha512-SWKaXD2QeQs06IE7qfJftsI5924Dqzp+V9xaa5RzZIEWhmlrG6Jt2iKwfgOPHu+5S8MEtOI7GdpKsXj46chXOw==
 
-emojibase-regex@^4.1.1:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-4.1.1.tgz#6e781aca520281600fe7a177f1582c33cf1fc545"
-  integrity sha512-KSigB1zQkNKFygLZ5bAfHs87LJa1ni8QTQtq8lc53Y74NF3Dk2r7kfa8MpooTO8JBb5Xz660X4tSjDB+I+7elA==
+emojibase-regex@^5.1.3:
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-5.1.3.tgz#f0ef621ed6ec624becd2326f999fd4ea01b94554"
+  integrity sha512-gT8T9LxLA8VJdI+8KQtyykB9qKzd7WuUL3M2yw6y9tplFeufOUANg3UKVaKUvkMcRNvZsSElWhxcJrx8WPE12g==
 
 encoding@^0.1.11:
   version "0.1.13"