new: Add fetchEmojis and fetchShortcodes functions.

packages/core/src/constants.ts

@@ -85,6 +85,8 @@ export const SUPPORTED_LOCALES = [
   'zh-hant', // Chinese (Traditional)
 ];
 
+export const NON_LATIN_LOCALES = ['ja', 'ko', 'ru', 'th', 'uk', 'zh', 'zh-hant'];
+
 // Special options for emoticon permutations.
 
 export const EMOTICON_OPTIONS: { [emoticon: string]: PermutationOptions } = {

packages/core/src/fetchEmojis.ts

@@ -0,0 +1,33 @@
+import { ShortcodePreset, CompactEmoji, Emoji } from './types';
+import fetchFromCDN, { FetchFromCDNOptions } from './fetchFromCDN';
+import fetchShortcodes from './fetchShortcodes';
+
+export interface FetchEmojisOptions extends FetchFromCDNOptions {
+  compact?: boolean;
+  shortcodes?: ShortcodePreset[];
+}
+
+// Compact
+async function fetchEmojis(
+  locale: string,
+  options: FetchEmojisOptions & { compact: true },
+): Promise<CompactEmoji[]>;
+
+// Full
+async function fetchEmojis(
+  locale: string,
+  options?: FetchEmojisOptions & { compact: false },
+): Promise<Emoji[]>;
+
+async function fetchEmojis(locale: string, options: FetchEmojisOptions = {}) {
+  const { compact = false, shortcodes: presets = [], ...opts } = options;
+  const emojis = await fetchFromCDN(`${locale}/${compact ? 'compact' : 'data'}.json`, opts);
+
+  if (presets.length > 0) {
+    await Promise.all(presets.map((preset) => fetchShortcodes(locale, preset, opts)));
+  }
+
+  return emojis;
+}
+
+export default fetchEmojis;

packages/core/src/fetchFromCDN.ts

@@ -6,7 +6,7 @@ export interface FetchFromCDNOptions extends RequestInit {
 export default function fetchFromCDN<T>(
   path: string,
   options: FetchFromCDNOptions = {},
-): Promise<T[]> {
+): Promise<T> {
   const { local = false, version = 'latest', ...opts } = options;
 
   if (__DEV__) {
@@ -25,11 +25,7 @@ export default function fetchFromCDN<T>(
 
   // Check the cache first
   if (cachedData) {
-    try {
-      return Promise.resolve(JSON.parse(cachedData));
-    } catch (error) {
-      return Promise.resolve([]);
-    }
+    return Promise.resolve(JSON.parse(cachedData));
   }
 
   return fetch(`https://cdn.jsdelivr.net/npm/emojibase-data@${version}/${path}`, {

packages/core/src/fetchShortcodes.ts

@@ -0,0 +1,15 @@
+import { ShortcodePreset, ShortcodesDataset } from './types';
+import fetchFromCDN, { FetchFromCDNOptions } from './fetchFromCDN';
+import { NON_LATIN_LOCALES } from './constants';
+
+export default function fetchShortcodes(
+  locale: string,
+  preset: ShortcodePreset,
+  options?: FetchFromCDNOptions,
+): Promise<ShortcodesDataset> {
+  if (preset === 'cldr-native' && !NON_LATIN_LOCALES.includes(locale)) {
+    return Promise.resolve({});
+  }
+
+  return fetchFromCDN(`${locale}/shortcodes/${preset}.json`, options);
+}

packages/core/src/index.ts

@@ -3,7 +3,9 @@
  * @license     https://opensource.org/licenses/MIT
  */
 
+import fetchEmojis from './fetchEmojis';
 import fetchFromCDN from './fetchFromCDN';
+import fetchShortcodes from './fetchShortcodes';
 import flattenEmojiData from './flattenEmojiData';
 import fromCodepointToUnicode from './fromCodepointToUnicode';
 import fromHexcodeToCodepoint from './fromHexcodeToCodepoint';
@@ -12,7 +14,9 @@ import generateEmoticonPermutations from './generateEmoticonPermutations';
 import stripHexcode from './stripHexcode';
 
 export {
+  fetchEmojis,
   fetchFromCDN,
+  fetchShortcodes,
   flattenEmojiData,
   fromCodepointToUnicode,
   fromHexcodeToCodepoint,

packages/core/src/types.ts

@@ -28,6 +28,8 @@ export type PresentationKey = 'text' | 'emoji';
 
 export type Shortcode = string;
 
+export type ShortcodePreset = 'cldr' | 'cldr-native';
+
 export type SkinTone = 1 | 2 | 3 | 4 | 5;
 
 export type SkinToneKey = 'light' | 'medium-light' | 'medium' | 'medium-dark' | 'dark';

packages/core/tests/fetchEmojis.test.ts

@@ -0,0 +1,34 @@
+import fetchEmojis from '../src/fetchEmojis';
+import { setupFetch } from './helpers';
+
+describe('fetchEmojis()', () => {
+  beforeEach(() => {
+    setupFetch();
+  });
+
+  it('triggers a fetch', async () => {
+    await fetchEmojis('de');
+
+    expect(global.fetch).toHaveBeenCalledWith(
+      'https://cdn.jsdelivr.net/npm/emojibase-data@latest/de/data.json',
+      {
+        credentials: 'omit',
+        mode: 'cors',
+        redirect: 'error',
+      },
+    );
+  });
+
+  it('triggers a fetch (compact)', async () => {
+    await fetchEmojis('ko', { compact: true, version: '1.2.3' });
+
+    expect(global.fetch).toHaveBeenCalledWith(
+      'https://cdn.jsdelivr.net/npm/emojibase-data@1.2.3/ko/compact.json',
+      {
+        credentials: 'omit',
+        mode: 'cors',
+        redirect: 'error',
+      },
+    );
+  });
+});

packages/core/tests/fetchFromCDN.test.ts

@@ -1,49 +1,9 @@
-/* eslint-disable @typescript-eslint/no-namespace */
-
-// @ts-expect-error
-import regeneratorRuntime from 'regenerator-runtime';
 import fetchFromCDN from '../src/fetchFromCDN';
-
-declare global {
-  namespace NodeJS {
-    interface Global {
-      fetch: any;
-      sessionStorage: any;
-      localStorage: any;
-      regeneratorRuntime: any;
-    }
-  }
-}
-
-global.regeneratorRuntime = regeneratorRuntime;
+import { setupFetch } from './helpers';
 
 describe('fetchFromCDN()', () => {
   beforeEach(() => {
-    Object.defineProperty(global, 'fetch', {
-      value: jest.fn(() =>
-        Promise.resolve({
-          json: () => [1, 2, 3],
-          ok: true,
-        }),
-      ),
-      configurable: true,
-    });
-
-    Object.defineProperty(global, 'sessionStorage', {
-      value: {
-        getItem: jest.fn(),
-        setItem: jest.fn(),
-      },
-      configurable: true,
-    });
-
-    Object.defineProperty(global, 'localStorage', {
-      value: {
-        getItem: jest.fn(),
-        setItem: jest.fn(),
-      },
-      configurable: true,
-    });
+    setupFetch();
   });
 
   it('errors if no path', () => {

packages/core/tests/fetchShortcodes.test.ts

@@ -0,0 +1,31 @@
+import fetchShortcodes from '../src/fetchShortcodes';
+import { setupFetch } from './helpers';
+
+describe('fetchShortcodes()', () => {
+  beforeEach(() => {
+    setupFetch({ '0000': 'shortcode' });
+  });
+
+  it('triggers a fetch', async () => {
+    await fetchShortcodes('de', 'cldr');
+
+    expect(global.fetch).toHaveBeenCalledWith(
+      'https://cdn.jsdelivr.net/npm/emojibase-data@latest/de/shortcodes/cldr.json',
+      {
+        credentials: 'omit',
+        mode: 'cors',
+        redirect: 'error',
+      },
+    );
+  });
+
+  it('returns an empty dataset for `cldr-native` and an invalid locale', async () => {
+    const res1 = await fetchShortcodes('ja', 'cldr-native');
+
+    expect(res1).not.toEqual({});
+
+    const res2 = await fetchShortcodes('en', 'cldr-native');
+
+    expect(res2).toEqual({});
+  });
+});

packages/core/tests/helpers.ts

@@ -0,0 +1,45 @@
+/* eslint-disable */
+
+// @ts-expect-error
+import regeneratorRuntime from 'regenerator-runtime';
+
+declare global {
+  namespace NodeJS {
+    interface Global {
+      fetch: any;
+      sessionStorage: any;
+      localStorage: any;
+      regeneratorRuntime: any;
+    }
+  }
+}
+
+export function setupFetch(response: unknown = [1, 2, 3]) {
+  global.regeneratorRuntime = regeneratorRuntime;
+
+  Object.defineProperty(global, 'fetch', {
+    configurable: true,
+    value: jest.fn(() =>
+      Promise.resolve({
+        json: () => response,
+        ok: true,
+      }),
+    ),
+  });
+
+  Object.defineProperty(global, 'sessionStorage', {
+    configurable: true,
+    value: {
+      getItem: jest.fn(),
+      setItem: jest.fn(),
+    },
+  });
+
+  Object.defineProperty(global, 'localStorage', {
+    configurable: true,
+    value: {
+      getItem: jest.fn(),
+      setItem: jest.fn(),
+    },
+  });
+}

packages/generator/src/generators/generateShortcodes.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment, unicorn/better-regex */
 
-import { SUPPORTED_LOCALES } from 'emojibase';
+import { SUPPORTED_LOCALES, NON_LATIN_LOCALES } from 'emojibase';
 import Kuroshiro from 'kuroshiro';
 import KuromojiAnalyzer from 'kuroshiro-analyzer-kuromoji';
 import { transliterate } from 'transliteration';
@@ -16,28 +16,6 @@ const CUSTOM_SHORTCODES: { [key: string]: string } = {
   t_rex: 'trex',
 };
 
-// Non-latin: ja, ko, ru, th, uk, zh, zh-hant
-const LATIN_LOCALES = new Set([
-  'da',
-  'de',
-  'en',
-  'en-gb',
-  'es',
-  'es-mx',
-  'et',
-  'fi',
-  'fr',
-  'hu',
-  'it',
-  'lt',
-  'ms',
-  'nb',
-  'nl',
-  'pl',
-  'pt',
-  'sv',
-]);
-
 const kuroshiro = new Kuroshiro();
 
 async function slugify(value: string, locale: string, transform: boolean = false): Promise<string> {
@@ -88,7 +66,7 @@ export default async function generateShortcodes(): Promise<void> {
   // Generate CLDR shortcodes for each locale
   await Promise.all(
     SUPPORTED_LOCALES.map(async (locale: string) => {
-      const isLatinChars = LATIN_LOCALES.has(locale);
+      const isLatinChars = !NON_LATIN_LOCALES.includes(locale);
       const annotations = await buildAnnotationData(locale);
       const cldr: ShortcodeDataMap = {};
       const cldrNonLatin: ShortcodeDataMap = {};