migrate emoji types

app/soapbox/actions/compose.ts

@@ -4,7 +4,7 @@ import { defineMessages, IntlShape } from 'react-intl';
 
 import snackbar from 'soapbox/actions/snackbar';
 import api from 'soapbox/api';
-import { search as emojiSearch } from 'soapbox/features/emoji/emoji_mart_search_light';
+import emojiSearch from 'soapbox/features/emoji/search';
 import { tagHistory } from 'soapbox/settings';
 import { isLoggedIn } from 'soapbox/utils/auth';
 import { getFeatures, parseVersion } from 'soapbox/utils/features';
@@ -20,8 +20,8 @@ import { getSettings } from './settings';
 import { createStatus } from './statuses';
 
 import type { History } from 'history';
-import type { Emoji } from 'soapbox/components/autosuggest_emoji';
 import type { AutoSuggestion } from 'soapbox/components/autosuggest_input';
+import type { Emoji } from 'soapbox/features/emoji';
 import type { AppDispatch, RootState } from 'soapbox/store';
 import type { Account, APIEntity, Status } from 'soapbox/types/entities';
 

app/soapbox/actions/emojis.ts

@@ -1,6 +1,6 @@
 import { saveSettings } from './settings';
 
-import type { Emoji } from 'soapbox/components/autosuggest_emoji';
+import type { Emoji } from 'soapbox/features/emoji';
 import type { AppDispatch } from 'soapbox/store';
 
 const EMOJI_USE = 'EMOJI_USE';

app/soapbox/components/autosuggest_emoji.tsx

@@ -1,17 +1,10 @@
 import React from 'react';
 
-import unicodeMapping from 'soapbox/features/emoji/emoji_unicode_mapping_light';
+import type { Emoji } from 'soapbox/features/emoji';
+import unicodeMapping from 'soapbox/features/emoji/mapping';
 import { joinPublicPath } from 'soapbox/utils/static';
 
-export type Emoji = {
-  id: string,
-  custom: boolean,
-  imageUrl: string,
-  native: string,
-  colons: string,
-}
-
-type UnicodeMapping = {
+interface UnicodeMapping {
   filename: string,
 }
 
@@ -25,14 +18,13 @@ const AutosuggestEmoji: React.FC<IAutosuggestEmoji> = ({ emoji }) => {
   if (emoji.custom) {
     url = emoji.imageUrl;
   } else {
-    // @ts-ignore
-    const mapping: UnicodeMapping = unicodeMapping[emoji.native] || unicodeMapping[emoji.native.replace(/\uFE0F$/, '')];
+    const mapping = unicodeMapping[emoji.native] || unicodeMapping[emoji.native.replace(/\uFE0F$/, '')];
 
     if (!mapping) {
       return null;
     }
 
-    url = joinPublicPath(`packs/emoji/${mapping.filename}.svg`);
+    url = joinPublicPath(`packs/emoji/${mapping.unified}.svg`);
   }
 
   return (

app/soapbox/components/autosuggest_input.tsx

@@ -3,11 +3,12 @@ import { List as ImmutableList } from 'immutable';
 import React from 'react';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 
-import AutosuggestEmoji, { Emoji } from 'soapbox/components/autosuggest_emoji';
+import AutosuggestEmoji from 'soapbox/components/autosuggest_emoji';
 import Icon from 'soapbox/components/icon';
 import AutosuggestAccount from 'soapbox/features/compose/components/autosuggest_account';
 import { isRtl } from 'soapbox/rtl';
 
+import type { Emoji } from 'soapbox/features/emoji';
 import type { Menu, MenuItem } from 'soapbox/components/dropdown_menu';
 
 type CursorMatch = [

app/soapbox/features/chats/components/chat-message-list.tsx

@@ -15,7 +15,7 @@ import { openModal } from 'soapbox/actions/modals';
 import { initReportById } from 'soapbox/actions/reports';
 import { Text } from 'soapbox/components/ui';
 import DropdownMenuContainer from 'soapbox/containers/dropdown_menu_container';
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import Bundle from 'soapbox/features/ui/components/bundle';
 import { MediaGallery } from 'soapbox/features/ui/util/async-components';
 import { useAppSelector, useAppDispatch, useRefEventHandler } from 'soapbox/hooks';

app/soapbox/features/chats/components/chat.tsx

@@ -5,7 +5,7 @@ import Avatar from 'soapbox/components/avatar';
 import DisplayName from 'soapbox/components/display-name';
 import Icon from 'soapbox/components/icon';
 import { Counter } from 'soapbox/components/ui';
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { useAppSelector } from 'soapbox/hooks';
 import { makeGetChat } from 'soapbox/selectors';
 
@@ -30,7 +30,9 @@ const Chat: React.FC<IChat> = ({ chatId, onClick }) => {
   const content = chat.getIn(['last_message', 'content']);
   const attachment = chat.getIn(['last_message', 'attachment']);
   const image = attachment && (attachment as any).getIn(['pleroma', 'mime_type'], '').startsWith('image/');
-  const parsedContent = content ? emojify(content) : '';
+
+  // TODO: fix chat.getIn typings
+  const parsedContent = content ? emojify(content as string) : '';
 
   return (
     <div className='account'>

app/soapbox/features/emoji/components/emoji_picker_dropdown.tsx

@@ -8,11 +8,12 @@ import { usePopper } from 'react-popper';
 import { IconButton } from 'soapbox/components/ui';
 import { useSettings } from 'soapbox/hooks';
 
-import { buildCustomEmojis } from '../../emoji/emoji';
+import { buildCustomEmojis } from '../../emoji';
 import { EmojiPicker as EmojiPickerAsync } from '../../ui/util/async-components';
-// import EmojiPicker from '../../emoji/emoji_picker';
+// import { Picker as EmojiPicker } from '../../emoji/emoji_picker';
 
 import type { List } from 'immutable';
+import type { Emoji } from 'soapbox/features/emoji';
 
 let EmojiPicker: any; // load asynchronously
 
@@ -37,7 +38,7 @@ interface IEmojiPickerDropdown {
   custom_emojis: List<any>,
   frequentlyUsedEmojis: string[],
   intl: any,
-  onPickEmoji: (emoji: any) => void,
+  onPickEmoji: (emoji: Emoji) => void,
   onSkinTone: () => void,
   skinTone: () => void,
 }
@@ -72,7 +73,8 @@ const EmojiPickerDropdown: React.FC<IEmojiPickerDropdown> = ({ custom_emojis, fr
     }
   };
 
-  const handlePick = (emoji: any) => {
+  const handlePick = (emoji: Emoji) => {
+    // TODO: remove me
     if (!emoji.native) {
       emoji.native = emoji.shortcodes;
     }

app/soapbox/features/emoji/data.ts

@@ -0,0 +1,3 @@
+import data from '@emoji-mart/data/sets/14/twitter.json';
+
+export default data;

app/soapbox/features/emoji/emoji_mart_data_light.ts

@@ -1,51 +0,0 @@
-// The output of this module is designed to mimic emoji-mart's
-// "data" object, such that we can use it for a light version of emoji-mart's
-// emojiIndex.search functionality.
-// import emojiCompressed from './emoji_compressed';
-import { unicodeToUnifiedName } from './unicode_to_unified_name';
-
-const [ shortCodesToEmojiData, skins, categories, short_names ] = [
-  [],
-  [],
-  [],
-  [],
-];
-
-const emojis: Record<string, any> = {};
-
-// decompress
-Object.keys(shortCodesToEmojiData).forEach((shortCode) => {
-  // @ts-ignore
-  const [
-    _filenameData, // eslint-disable-line @typescript-eslint/no-unused-vars
-    searchData,
-  // @ts-ignore
-  ] = shortCodesToEmojiData[shortCode];
-  const [
-    native,
-    short_names,
-    search,
-    unified,
-  ] = searchData;
-
-  emojis[shortCode] = {
-    native,
-    search,
-    short_names: [shortCode].concat(short_names),
-    unified: unified || unicodeToUnifiedName(native),
-  };
-});
-
-export {
-  emojis,
-  skins,
-  categories,
-  short_names,
-};
-
-export default {
-  emojis,
-  skins,
-  categories,
-  short_names,
-};

app/soapbox/features/emoji/index.ts

@@ -1,35 +1,21 @@
-import data from '@emoji-mart/data';
+// import data from '@emoji-mart/data';
 import { load as cheerioLoad } from 'cheerio';
 import { parseDocument } from 'htmlparser2';
 
-import type EmojiData from '@emoji-mart/data';
+import unicodeMapping from './mapping';
+
 import type { Node as CheerioNode } from 'cheerio';
+import type { Emoji as EmojiMartEmoji } from 'emoji-mart';
 
-interface IUniMap {
-  [s: string]: {
-    unified: string,
-    shortcode: string,
-  }
-}
+// export interface Emoji {
+//   id: string,
+//   custom: boolean,
+//   imageUrl: string,
+//   native: string,
+//   colons: string,
+// }
 
-const generateMappings = (data: typeof EmojiData): IUniMap => {
-  const result = {};
-  const emojis = Object.values(data.emojis ?? {});
-
-  for (const value of emojis) {
-    // @ts-ignore
-    for (const item of value.skins) {
-      const { unified, native } = item;
-
-      // @ts-ignore
-      result[native] = { unified, shortcode: value.id };
-    }
-  }
-
-  return result;
-};
-
-const unicodeMapping = generateMappings(data);
+export type Emoji = any;
 
 const isAlphaNumeric = (c: string) => {
   const code = c.charCodeAt(0);
@@ -59,11 +45,11 @@ const convertUnicode = (c: string) => {
   return `<img draggable="false" class="emojione" alt="${c}" title=":${shortcode}:" src="/packs/emoji/${unified}.svg">`;
 };
 
-const convertEmoji = (str: string, customEmojis: any, autoplay: boolean) => {
+const convertEmoji = (str: string, customEmojis: any) => {
   if (str.length < 3) return str;
   if (str in customEmojis) {
     const emoji = customEmojis[str];
-    const filename = autoplay ? emoji.url : emoji.static_url;
+    const filename = emoji.static_url;
 
     if (filename?.length > 0) {
       return convertCustom(str, filename);
@@ -73,29 +59,33 @@ const convertEmoji = (str: string, customEmojis: any, autoplay: boolean) => {
   return str;
 };
 
-const popStack = (stack: string, open: boolean, res: string) => {
-  res += stack;
+const popStack = (stack: string, open: boolean) => {
+  const ret = stack;
   open = false;
   stack = '';
+  return ret;
 };
 
-const transmogrify = (str: string, customEmojis = {}, autoplay: boolean) => {
+// TODO: handle grouped unicode emojis
+export const emojifyText = (str: string, customEmojis = {}) => {
   let res = '';
   let stack = '';
   let open = false;
 
-  for (const c of Array.from(str)) { // Array.from respects unicode
+  for (const c of Array.from(str)) { // chunk by unicode codepoint with Array.from
     if (c in unicodeMapping) {
       if (open) { // unicode emoji inside colon
-        popStack(stack, open, res);
+        res += popStack(stack, open);
       }
 
       res += convertUnicode(c);
+
     } else if (c === ':') {
       stack += ':';
 
+      // we see another : we convert it and clear the stack buffer
       if (open) {
-        res += convertEmoji(stack, customEmojis, autoplay);
+        res += convertEmoji(stack, customEmojis);
         stack = '';
       }
 
@@ -104,8 +94,10 @@ const transmogrify = (str: string, customEmojis = {}, autoplay: boolean) => {
       if (open) {
         stack += c;
 
+        // if the stack is non-null and we see invalid chars it's a string not emoji
+        // so we push it to the return result and clear it
         if (!validEmojiChar(c)) {
-          popStack(stack, open, res);
+          res += popStack(stack, open);
         }
       } else {
         res += c;
@@ -113,14 +105,26 @@ const transmogrify = (str: string, customEmojis = {}, autoplay: boolean) => {
     }
   }
 
+  // never found a closing colon so it's just a raw string
+  if (open) {
+    res += stack;
+  }
+
   return res;
 };
 
+// const parseHmtl = (str: string) => {
+//   const ret = [];
+//   let depth = 0;
+//
+//   return ret;
+// }
+
 const filterTextNodes = (idx: number, el: CheerioNode) => {
   return el.nodeType === Node.TEXT_NODE;
 };
 
-const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
+const emojify = (str: string, customEmojis = {}) => {
   const dom = parseDocument(str);
   const $ = cheerioLoad(dom, {
     xmlMode: true,
@@ -128,8 +132,8 @@ const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
   });
 
   $.root()
-    .contents()
-    .filter(filterTextNodes)
+    .contents() // iterate over flat map of all html elements
+    .filter(filterTextNodes) // only iterate over text nodes
     .each((idx, el) => {
       // skip common case
       // @ts-ignore
@@ -137,7 +141,7 @@ const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
 
       // mutating el.data is incorrect but we do it to prevent a second dom parse
       // @ts-ignore
-      el.data = transmogrify(el.data, customEmojis, autoplay);
+      el.data = emojifyText(el.data, customEmojis);
     });
 
   return $.html();
@@ -145,12 +149,12 @@ const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
 
 export default emojify;
 
-export const buildCustomEmojis = (customEmojis: any, autoplay = false) => {
-  const emojis: any[] = [];
+export const buildCustomEmojis = (customEmojis: any) => {
+  const emojis: EmojiMartEmoji[] = [];
 
   customEmojis.forEach((emoji: any) => {
     const shortcode = emoji.get('shortcode');
-    const url       = autoplay ? emoji.get('url') : emoji.get('static_url');
+    const url       = emoji.get('static_url');
     const name      = shortcode.replace(':', '');
 
     emojis.push({

app/soapbox/features/emoji/mapping.ts

@@ -0,0 +1,31 @@
+import emojiData from './data';
+
+import type EmojiData from '@emoji-mart/data';
+
+interface IUniMap {
+  [s: string]: {
+    unified: string,
+    shortcode: string,
+  }
+}
+
+export const generateMappings = (data: typeof EmojiData): IUniMap => {
+  const result = {};
+  const emojis = Object.values(data.emojis ?? {});
+
+  for (const value of emojis) {
+    // @ts-ignore
+    for (const item of value.skins) {
+      const { unified, native } = item;
+
+      // @ts-ignore
+      result[native] = { unified, shortcode: value.id };
+    }
+  }
+
+  return result;
+};
+
+const unicodeMapping = generateMappings(emojiData);
+
+export default unicodeMapping;

app/soapbox/features/emoji/search.ts

@@ -0,0 +1,17 @@
+export interface searchOptions {
+  maxResults?: number;
+}
+
+export interface Emoji {
+
+}
+
+export const addCustomToPool = (customEmojis: Emoji[]) => {
+};
+
+const search = (str: string, options: searchOptions) => {
+  console.log(str, options);
+  return [];
+};
+
+export default search;

app/soapbox/features/ui/components/link_footer.tsx

@@ -6,7 +6,7 @@ import { Link } from 'react-router-dom';
 
 import { logOut } from 'soapbox/actions/auth';
 import { Text } from 'soapbox/components/ui';
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { useSoapboxConfig, useOwnAccount, useFeatures } from 'soapbox/hooks';
 import sourceCode from 'soapbox/utils/code';
 

app/soapbox/normalizers/account.ts

@@ -11,7 +11,7 @@ import {
   fromJS,
 } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeEmoji } from 'soapbox/normalizers/emoji';
 import { unescapeHTML } from 'soapbox/utils/html';
 import { mergeDefined, makeEmojiMap } from 'soapbox/utils/normalizers';

app/soapbox/normalizers/poll.ts

@@ -11,7 +11,7 @@ import {
   fromJS,
 } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeEmoji } from 'soapbox/normalizers/emoji';
 import { makeEmojiMap } from 'soapbox/utils/normalizers';
 

app/soapbox/normalizers/status_edit.ts

@@ -9,7 +9,7 @@ import {
   fromJS,
 } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeAttachment } from 'soapbox/normalizers/attachment';
 import { normalizeEmoji } from 'soapbox/normalizers/emoji';
 import { normalizePoll } from 'soapbox/normalizers/poll';

app/soapbox/reducers/compose.ts

@@ -59,7 +59,7 @@ import { normalizeAttachment } from '../normalizers/attachment';
 import { unescapeHTML } from '../utils/html';
 
 import type { AnyAction } from 'redux';
-import type { Emoji } from 'soapbox/components/autosuggest_emoji';
+import type { Emoji } from 'soapbox/features/emoji';
 import type {
   Account as AccountEntity,
   APIEntity,

app/soapbox/reducers/custom_emojis.ts

@@ -1,10 +1,10 @@
 import { List as ImmutableList, Map as ImmutableMap, fromJS } from 'immutable';
 
-import { emojis as emojiData } from 'soapbox/features/emoji/emoji_mart_data_light';
-import { addCustomToPool } from 'soapbox/features/emoji/emoji_mart_search_light';
+import { buildCustomEmojis } from 'soapbox/features/emoji';
+import { addCustomToPool } from 'soapbox/features/emoji/search';
+// import emojiData from 'soapbox/features/emoji/data';
 
 import { CUSTOM_EMOJIS_FETCH_SUCCESS } from '../actions/custom_emojis';
-import { buildCustomEmojis } from '../features/emoji/emoji';
 
 import type { AnyAction } from 'redux';
 import type { APIEntity } from 'soapbox/types/entities';
@@ -17,14 +17,18 @@ const autosuggestPopulate = (emojis: ImmutableList<ImmutableMap<string, string>>
 };
 
 const importEmojis = (customEmojis: APIEntity[]) => {
-  const emojis = (fromJS(customEmojis) as ImmutableList<ImmutableMap<string, string>>).filter((emoji) => {
-    // If a custom emoji has the shortcode of a Unicode emoji, skip it.
-    // Otherwise it breaks EmojiMart.
-    // https://gitlab.com/soapbox-pub/soapbox-fe/-/issues/610
-    const shortcode = emoji.get('shortcode', '').toLowerCase();
-    return !emojiData[shortcode];
-  });
+  // const emojis = (fromJS(customEmojis)).filter((emoji) => {
+  //   // If a custom emoji has the shortcode of a Unicode emoji, skip it.
+  //   // Otherwise it breaks EmojiMart.
+  //   // https://gitlab.com/soapbox-pub/soapbox-fe/-/issues/610
+  //   const shortcode = emoji.get('shortcode', '').toLowerCase();
+  //   return !emojiData.emojis[shortcode];
+  // });
 
+  // @ts-ignore
+  const emojis = fromJS(customEmojis);
+
+  // @ts-ignore
   autosuggestPopulate(emojis);
   return emojis;
 };

app/soapbox/reducers/statuses.ts

@@ -1,7 +1,7 @@
 import escapeTextContentForBrowser from 'escape-html';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeStatus } from 'soapbox/normalizers';
 import { simulateEmojiReact, simulateUnEmojiReact } from 'soapbox/utils/emoji_reacts';
 import { stripCompatibilityFeatures, unescapeHTML } from 'soapbox/utils/html';

package.json

@@ -116,7 +116,7 @@
     "emoji-datasource": "5.0.0",
     "emoji-mart": "^5.1.0",
     "emoji-mart-old": "npm:emoji-mart-lazyload",
-    "entities": "^3.0.1",
+    "entities": "^4.3.1",
     "es6-symbol": "^3.1.1",
     "escape-html": "^1.0.3",
     "exif-js": "^2.3.0",

types/emoji-mart/index.d.ts

@@ -1,6 +1,17 @@
 declare module 'emoji-mart' {
-  export type PickerProps = {
-    custom?: { emojis: any[] }[],
+  export interface EmojiSkin {
+    src: string
+  }
+
+  export interface Emoji {
+    id: string,
+    name: string,
+    keywords: string[],
+    skins: EmojiSkin[],
+  }
+
+  export interface PickerProps {
+    custom?: { emojis: Emoji[] }[],
     set?: string,
     title?: string,
     theme?: string,

yarn.lock

@@ -5184,12 +5184,7 @@ entities@^2.0.0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
   integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
 
-entities@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/entities/-/entities-3.0.1.tgz#2b887ca62585e96db3903482d336c1006c3001d4"
-  integrity sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q==
-
-entities@^4.2.0, entities@^4.3.0:
+entities@^4.2.0, entities@^4.3.0, entities@^4.3.1:
   version "4.3.1"
   resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.1.tgz#c34062a94c865c322f9d67b4384e4169bcede6a4"
   integrity sha512-o4q/dYJlmyjP2zfnaWDUC6A3BQFmVTX+tZPezK7k0GLSU9QYCauscf5Y+qcEPzKL+EixVouYDgLQK5H9GrLpkg==