Skip to content

Commit

Permalink
Update Emojibase and switch to IamCal (Slack-style) shortcodes
Browse files Browse the repository at this point in the history 
for consistency with shortcodes commonly used by other platforms, as
discussed in element-hq/element-web#13857.

One thing to be aware of is that the currently used version of Twemoji
does not support a few of the newer emoji present in Emojibase, so these
look a little out of place in the emoji picker. Optimally Twemoji would
be updated at the same time, though I don't know how to do that.

Signed-off-by: Robin Townsend <robin@robin.town>
  • Loading branch information
@robintown
robintown committed Jul 11, 2021
1 parent 829169e commit cefd10d
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 79 deletions.
4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@
"counterpart": "^0.18.6",
"diff-dom": "^4.2.2",
"diff-match-patch": "^1.0.5",
"emojibase-data": "^5.1.1",
"emojibase-regex": "^4.1.1",
"emojibase-data": "^6.2.0",
"emojibase-regex": "^5.1.3",
"escape-html": "^1.0.3",
"file-saver": "^2.0.5",
"filesize": "6.1.0",
Expand Down
18 changes: 3 additions & 15 deletions src/HtmlUtils.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import { IExtendedSanitizeOptions } from './@types/sanitize-html';
import linkifyMatrix from './linkify-matrix';
import SettingsStore from './settings/SettingsStore';
import { tryTransformPermalinkToLocalHref } from "./utils/permalinks/Permalinks";
import { SHORTCODE_TO_EMOJI, getEmojiFromUnicode } from "./emoji";
import { getEmojiFromUnicode, getShortcodes } from "./emoji";
import ReplyThread from "./components/views/elements/ReplyThread";
import { mediaFromMxc } from "./customisations/Media";

Expand Down Expand Up @@ -78,20 +78,8 @@ function mightContainEmoji(str: string): boolean {
* @return {String} The shortcode (such as :thumbup:)
*/
export function unicodeToShortcode(char: string): string {
const data = getEmojiFromUnicode(char);
return (data && data.shortcodes ? `:${data.shortcodes[0]}:` : '');
}

/**
* Returns the unicode character for an emoji shortcode
*
* @param {String} shortcode The shortcode (such as :thumbup:)
* @return {String} The emoji character; null if none exists
*/
export function shortcodeToUnicode(shortcode: string): string {
shortcode = shortcode.slice(1, shortcode.length - 1);
const data = SHORTCODE_TO_EMOJI.get(shortcode);
return data ? data.unicode : null;
const shortcodes = getShortcodes(getEmojiFromUnicode(char));
return shortcodes.length > 0 ? `:${shortcodes[0]}:` : '';
}

export function processHtmlForSending(html: string): string {
Expand Down
69 changes: 36 additions & 33 deletions src/autocomplete/EmojiProvider.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ import { PillCompletion } from './Components';
import { ICompletion, ISelectionRange } from './Autocompleter';
import { uniq, sortBy } from 'lodash';
import SettingsStore from "../settings/SettingsStore";
import { shortcodeToUnicode } from '../HtmlUtils';
import { EMOJI, IEmoji } from '../emoji';
import { EMOJI, IEmoji, getShortcodes } from '../emoji';

import EMOTICON_REGEX from 'emojibase-regex/emoticon';

Expand All @@ -38,21 +37,26 @@ const EMOJI_REGEX = new RegExp('(' + EMOTICON_REGEX.source + '|(?:^|\\s):[+-\\w]

interface IEmojiShort {
emoji: IEmoji;
shortname: string;
shortcode: string;
altShortcodes: string[];
_orderBy: number;
}

const EMOJI_SHORTNAMES: IEmojiShort[] = EMOJI.sort((a, b) => {
const EMOJI_SHORTCODES: IEmojiShort[] = EMOJI.sort((a, b) => {
if (a.group === b.group) {
return a.order - b.order;
}
return a.group - b.group;
}).map((emoji, index) => ({
emoji,
shortname: `:${emoji.shortcodes[0]}:`,
// Include the index so that we can preserve the original order
_orderBy: index,
}));
}).map((emoji, index) => {
const [shortcode, ...altShortcodes] = getShortcodes(emoji);
return {
emoji,
shortcode: shortcode ? `:${shortcode}:` : undefined,
altShortcodes: altShortcodes.map(s => `:${s}:`),
// Include the index so that we can preserve the original order
_orderBy: index,
};
}).filter(emoji => emoji.shortcode);

function score(query, space) {
const index = space.indexOf(query);
Expand All @@ -69,15 +73,15 @@ export default class EmojiProvider extends AutocompleteProvider {

constructor() {
super(EMOJI_REGEX);
this.matcher = new QueryMatcher<IEmojiShort>(EMOJI_SHORTNAMES, {
keys: ['emoji.emoticon', 'shortname'],
this.matcher = new QueryMatcher<IEmojiShort>(EMOJI_SHORTCODES, {
keys: ['emoji.emoticon', 'shortcode'],
funcs: [
(o) => o.emoji.shortcodes.length > 1 ? o.emoji.shortcodes.slice(1).map(s => `:${s}:`).join(" ") : "", // aliases
o => o.altShortcodes.join(" "), // aliases
],
// For matching against ascii equivalents
shouldMatchWordsOnly: false,
});
this.nameMatcher = new QueryMatcher(EMOJI_SHORTNAMES, {
this.nameMatcher = new QueryMatcher(EMOJI_SHORTCODES, {
keys: ['emoji.annotation'],
// For removing punctuation
shouldMatchWordsOnly: true,
Expand Down Expand Up @@ -105,34 +109,33 @@ export default class EmojiProvider extends AutocompleteProvider {

const sorters = [];
// make sure that emoticons come first
sorters.push((c) => score(matchedString, c.emoji.emoticon || ""));
sorters.push(c => score(matchedString, c.emoji.emoticon || ""));

// then sort by score (Infinity if matchedString not in shortname)
sorters.push((c) => score(matchedString, c.shortname));
// then sort by score (Infinity if matchedString not in shortcode)
sorters.push(c => score(matchedString, c.shortcode));
// then sort by max score of all shortcodes, trim off the `:`
sorters.push((c) => Math.min(...c.emoji.shortcodes.map(s => score(matchedString.substring(1), s))));
// If the matchedString is not empty, sort by length of shortname. Example:
sorters.push(c => Math.min(
...[c.shortcode, ...c.altShortcodes].map(s => score(matchedString.substring(1), s)),
));
// If the matchedString is not empty, sort by length of shortcode. Example:
// matchedString = ":bookmark"
// completions = [":bookmark:", ":bookmark_tabs:", ...]
if (matchedString.length > 1) {
sorters.push((c) => c.shortname.length);
sorters.push(c => c.shortcode.length);
}
// Finally, sort by original ordering
sorters.push((c) => c._orderBy);
sorters.push(c => c._orderBy);
completions = sortBy(uniq(completions), sorters);

completions = completions.map(({ shortname }) => {
const unicode = shortcodeToUnicode(shortname);
return {
completion: unicode,
component: (
<PillCompletion title={shortname} aria-label={unicode}>
<span>{ unicode }</span>
</PillCompletion>
),
range,
};
}).slice(0, LIMIT);
completions = completions.map(c => ({
completion: c.emoji.unicode,
component: (
<PillCompletion title={c.shortcode} aria-label={c.emoji.unicode}>
<span>{ c.emoji.unicode }</span>
</PillCompletion>
),
range,
})).slice(0, LIMIT);
}
return completions;
}
Expand Down
12 changes: 6 additions & 6 deletions src/components/views/emojipicker/Preview.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ limitations under the License.

import React from 'react';

import { IEmoji } from "../../../emoji";
import { IEmoji, getShortcodes } from "../../../emoji";
import { replaceableComponent } from "../../../utils/replaceableComponent";

interface IProps {
Expand All @@ -30,8 +30,8 @@ class Preview extends React.PureComponent<IProps> {
const {
unicode = "",
annotation = "",
shortcodes: [shortcode = ""],
} = this.props.emoji || {};
} = this.props.emoji;
const shortcode = getShortcodes(this.props.emoji)[0];

return (
<div className="mx_EmojiPicker_footer mx_EmojiPicker_preview">
Expand All @@ -42,9 +42,9 @@ class Preview extends React.PureComponent<IProps> {
<div className="mx_EmojiPicker_name mx_EmojiPicker_preview_name">
{annotation}
</div>
<div className="mx_EmojiPicker_shortcode">
{shortcode}
</div>
{ shortcode ?
<div className="mx_EmojiPicker_shortcode">{shortcode}</div> :
null }
</div>
</div>
);
Expand Down
7 changes: 5 additions & 2 deletions src/components/views/emojipicker/QuickReactions.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ limitations under the License.
import React from 'react';

import { _t } from '../../../languageHandler';
import { getEmojiFromUnicode, IEmoji } from "../../../emoji";
import { getEmojiFromUnicode, getShortcodes, IEmoji } from "../../../emoji";
import Emoji from "./Emoji";
import { replaceableComponent } from "../../../utils/replaceableComponent";

Expand Down Expand Up @@ -62,14 +62,17 @@ class QuickReactions extends React.Component<IProps, IState> {
};

render() {
const shortcode = this.state.hover ? getShortcodes(this.state.hover)[0] : undefined;
return (
<section className="mx_EmojiPicker_footer mx_EmojiPicker_quick mx_EmojiPicker_category">
<h2 className="mx_EmojiPicker_quick_header mx_EmojiPicker_category_label">
{!this.state.hover
? _t("Quick Reactions")
: <React.Fragment>
<span className="mx_EmojiPicker_name">{this.state.hover.annotation}</span>
<span className="mx_EmojiPicker_shortcode">{this.state.hover.shortcodes[0]}</span>
{ shortcode ?
<span className="mx_EmojiPicker_shortcode">{shortcode}</span> :
null }
</React.Fragment>
}
</h2>
Expand Down
25 changes: 12 additions & 13 deletions src/emoji.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ limitations under the License.
*/

import EMOJIBASE from 'emojibase-data/en/compact.json';
import SHORTCODES from 'emojibase-data/en/shortcodes/iamcal.json';

export interface IEmoji {
annotation: string;
group: number;
group?: number;
hexcode: string;
order: number;
shortcodes: string[];
tags: string[];
order?: number;
tags?: string[];
unicode: string;
emoticon?: string;
}
Expand All @@ -34,10 +34,14 @@ interface IEmojiWithFilterString extends IEmoji {
// The unicode is stored without the variant selector
const UNICODE_TO_EMOJI = new Map<string, IEmojiWithFilterString>(); // not exported as gets for it are handled by getEmojiFromUnicode
export const EMOTICON_TO_EMOJI = new Map<string, IEmojiWithFilterString>();
export const SHORTCODE_TO_EMOJI = new Map<string, IEmojiWithFilterString>();

export const getEmojiFromUnicode = unicode => UNICODE_TO_EMOJI.get(stripVariation(unicode));

const toArray = (shortcodes?: string | string[]): string[] =>
typeof shortcodes === "string" ? [shortcodes] : (shortcodes ?? []);
export const getShortcodes = (emoji: IEmoji): string[] =>
toArray(SHORTCODES[emoji.hexcode]);

const EMOJIBASE_GROUP_ID_TO_CATEGORY = [
"people", // smileys
"people", // actually people
Expand Down Expand Up @@ -66,12 +70,14 @@ const ZERO_WIDTH_JOINER = "\u200D";

// Store various mappings from unicode/emoticon/shortcode to the Emoji objects
EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => {
const shortcodes = getShortcodes(emoji);
const categoryId = EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group];
if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) {
DATA_BY_CATEGORY[categoryId].push(emoji);
}

// This is used as the string to match the query against when filtering emojis
emoji.filterString = (`${emoji.annotation}\n${emoji.shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` +
emoji.filterString = (`${emoji.annotation}\n${shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` +
`${emoji.unicode.split(ZERO_WIDTH_JOINER).join("\n")}`).toLowerCase();

// Add mapping from unicode to Emoji object
Expand All @@ -87,13 +93,6 @@ EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => {
// Add mapping from emoticon to Emoji object
EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji);
}

if (emoji.shortcodes) {
// Add mapping from each shortcode to Emoji object
emoji.shortcodes.forEach(shortcode => {
SHORTCODE_TO_EMOJI.set(shortcode, emoji);
});
}
});

/**
Expand Down
16 changes: 8 additions & 8 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3022,15 +3022,15 @@ emoji-regex@^8.0.0:
resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==

emojibase-data@^5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-5.1.1.tgz#0a0d63dd07ce1376b3d27642f28cafa46f651de6"
integrity sha512-za/ma5SfogHjwUmGFnDbTvSfm8GGFvFaPS27GPti16YZSp5EPgz+UDsZCATXvJGit+oRNBbG/FtybXHKi2UQgQ==
emojibase-data@^6.2.0:
version "6.2.0"
resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-6.2.0.tgz#db6c75c36905284fa623f4aa5f468d2be6ed364a"
integrity sha512-SWKaXD2QeQs06IE7qfJftsI5924Dqzp+V9xaa5RzZIEWhmlrG6Jt2iKwfgOPHu+5S8MEtOI7GdpKsXj46chXOw==

emojibase-regex@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-4.1.1.tgz#6e781aca520281600fe7a177f1582c33cf1fc545"
integrity sha512-KSigB1zQkNKFygLZ5bAfHs87LJa1ni8QTQtq8lc53Y74NF3Dk2r7kfa8MpooTO8JBb5Xz660X4tSjDB+I+7elA==
emojibase-regex@^5.1.3:
version "5.1.3"
resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-5.1.3.tgz#f0ef621ed6ec624becd2326f999fd4ea01b94554"
integrity sha512-gT8T9LxLA8VJdI+8KQtyykB9qKzd7WuUL3M2yw6y9tplFeufOUANg3UKVaKUvkMcRNvZsSElWhxcJrx8WPE12g==

encoding@^0.1.11:
version "0.1.13"
Expand Down

0 comments on commit cefd10d

Please sign in to comment.