migrate emoji types

app/soapbox/actions/compose.ts

@@ -4,7 +4,7 @@ import { defineMessages, IntlShape } from 'react-intl';
 
 import snackbar from 'soapbox/actions/snackbar';
 import api from 'soapbox/api';
-import { search as emojiSearch } from 'soapbox/features/emoji/emoji_mart_search_light';
+import emojiSearch from 'soapbox/features/emoji/search';
 import { tagHistory } from 'soapbox/settings';
 import { isLoggedIn } from 'soapbox/utils/auth';
 import { getFeatures, parseVersion } from 'soapbox/utils/features';
@@ -20,8 +20,8 @@ import { getSettings } from './settings';
 import { createStatus } from './statuses';
 
 import type { History } from 'history';
-import type { Emoji } from 'soapbox/components/autosuggest_emoji';
 import type { AutoSuggestion } from 'soapbox/components/autosuggest_input';
+import type { Emoji } from 'soapbox/features/emoji';
 import type { AppDispatch, RootState } from 'soapbox/store';
 import type { Account, APIEntity, Status } from 'soapbox/types/entities';
 

app/soapbox/actions/emojis.ts

@@ -1,6 +1,6 @@
 import { saveSettings } from './settings';
 
-import type { Emoji } from 'soapbox/components/autosuggest_emoji';
+import type { Emoji } from 'soapbox/features/emoji';
 import type { AppDispatch } from 'soapbox/store';
 
 const EMOJI_USE = 'EMOJI_USE';

app/soapbox/components/autosuggest_emoji.tsx

@@ -1,17 +1,10 @@
 import React from 'react';
 
-import unicodeMapping from 'soapbox/features/emoji/emoji_unicode_mapping_light';
+import type { Emoji } from 'soapbox/features/emoji';
+import unicodeMapping from 'soapbox/features/emoji/mapping';
 import { joinPublicPath } from 'soapbox/utils/static';
 
-export type Emoji = {
+interface UnicodeMapping {
-  id: string,
-  custom: boolean,
-  imageUrl: string,
-  native: string,
-  colons: string,
-}
-
-type UnicodeMapping = {
   filename: string,
 }
 
@@ -25,14 +18,13 @@ const AutosuggestEmoji: React.FC<IAutosuggestEmoji> = ({ emoji }) => {
   if (emoji.custom) {
     url = emoji.imageUrl;
   } else {
-    // @ts-ignore
+    const mapping = unicodeMapping[emoji.native] || unicodeMapping[emoji.native.replace(/\uFE0F$/, '')];
-    const mapping: UnicodeMapping = unicodeMapping[emoji.native] || unicodeMapping[emoji.native.replace(/\uFE0F$/, '')];
 
     if (!mapping) {
       return null;
     }
 
-    url = joinPublicPath(`packs/emoji/${mapping.filename}.svg`);
+    url = joinPublicPath(`packs/emoji/${mapping.unified}.svg`);
   }
 
   return (

app/soapbox/components/autosuggest_input.tsx

@@ -3,11 +3,12 @@ import { List as ImmutableList } from 'immutable';
 import React from 'react';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 
-import AutosuggestEmoji, { Emoji } from 'soapbox/components/autosuggest_emoji';
+import AutosuggestEmoji from 'soapbox/components/autosuggest_emoji';
 import Icon from 'soapbox/components/icon';
 import AutosuggestAccount from 'soapbox/features/compose/components/autosuggest_account';
 import { isRtl } from 'soapbox/rtl';
 
+import type { Emoji } from 'soapbox/features/emoji';
 import type { Menu, MenuItem } from 'soapbox/components/dropdown_menu';
 
 type CursorMatch = [

app/soapbox/components/icon_button.js

@@ -4,7 +4,7 @@ import React from 'react';
 import spring from 'react-motion/lib/spring';
 
 import Icon from 'soapbox/components/icon';
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 
 import Motion from '../features/ui/util/optional_motion';
 

app/soapbox/features/chats/components/chat-message-list.tsx

@@ -15,7 +15,7 @@ import { openModal } from 'soapbox/actions/modals';
 import { initReportById } from 'soapbox/actions/reports';
 import { Text } from 'soapbox/components/ui';
 import DropdownMenuContainer from 'soapbox/containers/dropdown_menu_container';
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import Bundle from 'soapbox/features/ui/components/bundle';
 import { MediaGallery } from 'soapbox/features/ui/util/async-components';
 import { useAppSelector, useAppDispatch, useRefEventHandler } from 'soapbox/hooks';

app/soapbox/features/chats/components/chat.tsx

@@ -5,7 +5,7 @@ import Avatar from 'soapbox/components/avatar';
 import DisplayName from 'soapbox/components/display-name';
 import Icon from 'soapbox/components/icon';
 import { Counter } from 'soapbox/components/ui';
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { useAppSelector } from 'soapbox/hooks';
 import { makeGetChat } from 'soapbox/selectors';
 
@@ -30,7 +30,9 @@ const Chat: React.FC<IChat> = ({ chatId, onClick }) => {
   const content = chat.getIn(['last_message', 'content']);
   const attachment = chat.getIn(['last_message', 'attachment']);
   const image = attachment && (attachment as any).getIn(['pleroma', 'mime_type'], '').startsWith('image/');
-  const parsedContent = content ? emojify(content) : '';
+
+  // TODO: fix chat.getIn typings
+  const parsedContent = content ? emojify(content as string) : '';
 
   return (
     <div className='account'>

app/soapbox/features/compose/components/compose_form.js

@@ -14,11 +14,11 @@ import Icon from 'soapbox/components/icon';
 import { Button } from 'soapbox/components/ui';
 import { isMobile } from 'soapbox/is_mobile';
 
+import EmojiPickerDropdown from '../../emoji/containers/emoji_picker_dropdown_container';
 import PollForm from '../components/polls/poll-form';
 import ReplyMentions from '../components/reply_mentions';
 import UploadForm from '../components/upload_form';
 import Warning from '../components/warning';
-import EmojiPickerDropdown from '../containers/emoji_picker_dropdown_container';
 import MarkdownButtonContainer from '../containers/markdown_button_container';
 import PollButtonContainer from '../containers/poll_button_container';
 import PrivacyDropdownContainer from '../containers/privacy_dropdown_container';

app/soapbox/features/emoji/components/emoji_picker_dropdown.tsx

@@ -8,11 +8,12 @@ import { usePopper } from 'react-popper';
 import { IconButton } from 'soapbox/components/ui';
 import { useSettings } from 'soapbox/hooks';
 
-import { buildCustomEmojis } from '../../emoji/emoji';
+import { buildCustomEmojis } from '../../emoji';
 import { EmojiPicker as EmojiPickerAsync } from '../../ui/util/async-components';
-// import EmojiPicker from '../../emoji/emoji_picker';
+// import { Picker as EmojiPicker } from '../../emoji/emoji_picker';
 
 import type { List } from 'immutable';
+import type { Emoji } from 'soapbox/features/emoji';
 
 let EmojiPicker: any; // load asynchronously
 
@@ -37,7 +38,7 @@ interface IEmojiPickerDropdown {
   custom_emojis: List<any>,
   frequentlyUsedEmojis: string[],
   intl: any,
-  onPickEmoji: (emoji: any) => void,
+  onPickEmoji: (emoji: Emoji) => void,
   onSkinTone: () => void,
   skinTone: () => void,
 }
@@ -72,7 +73,8 @@ const EmojiPickerDropdown: React.FC<IEmojiPickerDropdown> = ({ custom_emojis, fr
     }
   };
 
-  const handlePick = (emoji: any) => {
+  const handlePick = (emoji: Emoji) => {
+    // TODO: remove me
     if (!emoji.native) {
       emoji.native = emoji.shortcodes;
     }

app/soapbox/features/emoji/data.ts

@@ -0,0 +1,3 @@
+import data from '@emoji-mart/data/sets/14/twitter.json';
+
+export default data;

app/soapbox/features/emoji/emoji_mart_data_light.ts

@@ -1,51 +0,0 @@
-// The output of this module is designed to mimic emoji-mart's
-// "data" object, such that we can use it for a light version of emoji-mart's
-// emojiIndex.search functionality.
-// import emojiCompressed from './emoji_compressed';
-import { unicodeToUnifiedName } from './unicode_to_unified_name';
-
-const [ shortCodesToEmojiData, skins, categories, short_names ] = [
-  [],
-  [],
-  [],
-  [],
-];
-
-const emojis: Record<string, any> = {};
-
-// decompress
-Object.keys(shortCodesToEmojiData).forEach((shortCode) => {
-  // @ts-ignore
-  const [
-    _filenameData, // eslint-disable-line @typescript-eslint/no-unused-vars
-    searchData,
-  // @ts-ignore
-  ] = shortCodesToEmojiData[shortCode];
-  const [
-    native,
-    short_names,
-    search,
-    unified,
-  ] = searchData;
-
-  emojis[shortCode] = {
-    native,
-    search,
-    short_names: [shortCode].concat(short_names),
-    unified: unified || unicodeToUnifiedName(native),
-  };
-});
-
-export {
-  emojis,
-  skins,
-  categories,
-  short_names,
-};
-
-export default {
-  emojis,
-  skins,
-  categories,
-  short_names,
-};

app/soapbox/features/emoji/emoji_mart_search_light.js

@@ -1,183 +0,0 @@
-// This code is largely borrowed from:
-// https://github.com/missive/emoji-mart/blob/5f2ffcc/src/utils/emoji-index.js
-
-import data from './emoji_mart_data_light';
-import { getData, getSanitizedData, uniq, intersect } from './emoji_utils';
-
-const originalPool = {};
-let index = {};
-const emojisList = {};
-const emoticonsList = {};
-let customEmojisList = [];
-
-for (const emoji in data.emojis) {
-  const emojiData = data.emojis[emoji];
-  const { short_names, emoticons } = emojiData;
-  const id = short_names[0];
-
-  if (emoticons) {
-    emoticons.forEach(emoticon => {
-      if (emoticonsList[emoticon]) {
-        return;
-      }
-
-      emoticonsList[emoticon] = id;
-    });
-  }
-
-  emojisList[id] = getSanitizedData(id);
-  originalPool[id] = emojiData;
-}
-
-function clearCustomEmojis(pool) {
-  customEmojisList.forEach((emoji) => {
-    const emojiId = emoji.id || emoji.short_names[0];
-
-    delete pool[emojiId];
-    delete emojisList[emojiId];
-  });
-}
-
-export function addCustomToPool(custom, pool = originalPool) {
-  if (customEmojisList.length) clearCustomEmojis(pool);
-
-  custom.forEach((emoji) => {
-    const emojiId = emoji.id || emoji.short_names[0];
-
-    if (emojiId && !pool[emojiId]) {
-      pool[emojiId] = getData(emoji);
-      emojisList[emojiId] = getSanitizedData(emoji);
-    }
-  });
-
-  customEmojisList = custom;
-  index = {};
-}
-
-export function search(value, { emojisToShowFilter, maxResults, include, exclude, custom } = {}) {
-  if (custom !== undefined) {
-    if (customEmojisList !== custom)
-      addCustomToPool(custom, originalPool);
-  } else {
-    custom = [];
-  }
-
-  maxResults = maxResults || 75;
-  include = include || [];
-  exclude = exclude || [];
-
-  let results = null,
-    pool = originalPool;
-
-  if (value.length) {
-    if (value === '-' || value === '-1') {
-      return [emojisList['-1']];
-    }
-
-    let values = value.toLowerCase().split(/[\s|,|\-|_]+/),
-      allResults = [];
-
-    if (values.length > 2) {
-      values = [values[0], values[1]];
-    }
-
-    if (include.length || exclude.length) {
-      pool = {};
-
-      data.categories.forEach(category => {
-        const isIncluded = include && include.length ? include.indexOf(category.name.toLowerCase()) > -1 : true;
-        const isExcluded = exclude && exclude.length ? exclude.indexOf(category.name.toLowerCase()) > -1 : false;
-        if (!isIncluded || isExcluded) {
-          return;
-        }
-
-        category.emojis.forEach(emojiId => pool[emojiId] = data.emojis[emojiId]);
-      });
-
-      if (custom.length) {
-        const customIsIncluded = include && include.length ? include.indexOf('custom') > -1 : true;
-        const customIsExcluded = exclude && exclude.length ? exclude.indexOf('custom') > -1 : false;
-        if (customIsIncluded && !customIsExcluded) {
-          addCustomToPool(custom, pool);
-        }
-      }
-    }
-
-    const searchValue = (value) => {
-      let aPool = pool,
-        aIndex = index,
-        length = 0;
-
-      for (let charIndex = 0; charIndex < value.length; charIndex++) {
-        const char = value[charIndex];
-        length++;
-
-        aIndex[char] = aIndex[char] || {};
-        aIndex = aIndex[char];
-
-        if (!aIndex.results) {
-          const scores = {};
-
-          aIndex.results = [];
-          aIndex.pool = {};
-
-          for (const id in aPool) {
-            const emoji = aPool[id],
-              { search } = emoji,
-              sub = value.substr(0, length),
-              subIndex = search.indexOf(sub);
-
-            if (subIndex !== -1) {
-              let score = subIndex + 1;
-              if (sub === id) score = 0;
-
-              aIndex.results.push(emojisList[id]);
-              aIndex.pool[id] = emoji;
-
-              scores[id] = score;
-            }
-          }
-
-          aIndex.results.sort((a, b) => {
-            const aScore = scores[a.id],
-              bScore = scores[b.id];
-
-            return aScore - bScore;
-          });
-        }
-
-        aPool = aIndex.pool;
-      }
-
-      return aIndex.results;
-    };
-
-    if (values.length > 1) {
-      results = searchValue(value);
-    } else {
-      results = [];
-    }
-
-    allResults = values.map(searchValue).filter(a => a);
-
-    if (allResults.length > 1) {
-      allResults = intersect.apply(null, allResults);
-    } else if (allResults.length) {
-      allResults = allResults[0];
-    }
-
-    results = uniq(results.concat(allResults));
-  }
-
-  if (results) {
-    if (emojisToShowFilter) {
-      results = results.filter((result) => emojisToShowFilter(data.emojis[result.id]));
-    }
-
-    if (results && results.length > maxResults) {
-      results = results.slice(0, maxResults);
-    }
-  }
-
-  return results;
-}

app/soapbox/features/emoji/emoji_unicode_mapping_light.js

@@ -1,39 +0,0 @@
-// A mapping of unicode strings to an object containing the filename
-// (i.e. the svg filename) and a shortCode intended to be shown
-// as a "title" attribute in an HTML element (aka tooltip).
-
-const [
-  shortCodesToEmojiData,
-  skins, // eslint-disable-line @typescript-eslint/no-unused-vars
-  categories, // eslint-disable-line @typescript-eslint/no-unused-vars
-  short_names, // eslint-disable-line @typescript-eslint/no-unused-vars
-  emojisWithoutShortCodes,
-] = [
-  [],
-  [],
-  [],
-  [],
-  [],
-];
-
-const { unicodeToFilename } = require('./unicode_to_filename');
-
-// decompress
-const unicodeMapping = {};
-
-function processEmojiMapData(emojiMapData, shortCode) {
-  const [ native, filename ] = emojiMapData;
-
-  unicodeMapping[native] = {
-    shortCode,
-    filename: filename || unicodeToFilename(native),
-  };
-}
-
-Object.keys(shortCodesToEmojiData).forEach((shortCode) => {
-  const [ filenameData ] = shortCodesToEmojiData[shortCode];
-  filenameData.forEach(emojiMapData => processEmojiMapData(emojiMapData, shortCode));
-});
-emojisWithoutShortCodes.forEach(emojiMapData => processEmojiMapData(emojiMapData));
-
-module.exports = unicodeMapping;

app/soapbox/features/emoji/emoji_utils.js

@@ -1,253 +0,0 @@
-// This code is largely borrowed from:
-// https://github.com/missive/emoji-mart/blob/5f2ffcc/src/utils/index.js
-
-import data from './emoji_mart_data_light';
-
-const buildSearch = (data) => {
-  const search = [];
-
-  const addToSearch = (strings, split) => {
-    if (!strings) {
-      return;
-    }
-
-    (Array.isArray(strings) ? strings : [strings]).forEach((string) => {
-      (split ? string.split(/[-|_|\s]+/) : [string]).forEach((s) => {
-        s = s.toLowerCase();
-
-        if (search.indexOf(s) === -1) {
-          search.push(s);
-        }
-      });
-    });
-  };
-
-  addToSearch(data.short_names, true);
-  addToSearch(data.name, true);
-  addToSearch(data.keywords, false);
-  addToSearch(data.emoticons, false);
-
-  return search.join(',');
-};
-
-const _String = String;
-
-const stringFromCodePoint = _String.fromCodePoint || function() {
-  const MAX_SIZE = 0x4000;
-  const codeUnits = [];
-  let highSurrogate;
-  let lowSurrogate;
-  let index = -1;
-  const length = arguments.length;
-  if (!length) {
-    return '';
-  }
-  let result = '';
-  while (++index < length) {
-    let codePoint = Number(arguments[index]);
-    if (
-      !isFinite(codePoint) ||       // `NaN`, `+Infinity`, or `-Infinity`
-      codePoint < 0 ||              // not a valid Unicode code point
-      codePoint > 0x10FFFF ||       // not a valid Unicode code point
-      Math.floor(codePoint) !== codePoint // not an integer
-    ) {
-      throw RangeError('Invalid code point: ' + codePoint);
-    }
-    if (codePoint <= 0xFFFF) { // BMP code point
-      codeUnits.push(codePoint);
-    } else { // Astral code point; split in surrogate halves
-      // http://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
-      codePoint -= 0x10000;
-      highSurrogate = (codePoint >> 10) + 0xD800;
-      lowSurrogate = (codePoint % 0x400) + 0xDC00;
-      codeUnits.push(highSurrogate, lowSurrogate);
-    }
-    if (index + 1 === length || codeUnits.length > MAX_SIZE) {
-      result += String.fromCharCode.apply(null, codeUnits);
-      codeUnits.length = 0;
-    }
-  }
-  return result;
-};
-
-const _JSON = JSON;
-
-const COLONS_REGEX = /^(?:\:([^\:]+)\:)(?:\:skin-tone-(\d)\:)?$/;
-const SKINS = [
-  '1F3FA', '1F3FB', '1F3FC',
-  '1F3FD', '1F3FE', '1F3FF',
-];
-
-function unifiedToNative(unified) {
-  const unicodes = unified.split('-'),
-    codePoints = unicodes.map((u) => `0x${u}`);
-
-  return stringFromCodePoint.apply(null, codePoints);
-}
-
-function sanitize(emoji) {
-  const { name, short_names, skin_tone, skin_variations, emoticons, unified, custom, imageUrl } = emoji;
-  const id = emoji.id || short_names[0];
-  const colons = `:${id}:`;
-
-  if (custom) {
-    return {
-      id,
-      name,
-      colons,
-      emoticons,
-      custom,
-      imageUrl,
-    };
-  }
-
-  return {
-    id,
-    name,
-    colons: skin_tone ? `${colons}:skin-tone-${skin_tone}:` : colons,
-    emoticons,
-    unified: unified.toLowerCase(),
-    skin: skin_tone || (skin_variations ? 1 : null),
-    native: unifiedToNative(unified),
-  };
-}
-
-function getSanitizedData() {
-  return sanitize(getData(...arguments));
-}
-
-function getData(emoji, skin, set) {
-  let emojiData = {};
-
-  if (typeof emoji === 'string') {
-    const matches = emoji.match(COLONS_REGEX);
-
-    if (matches) {
-      emoji = matches[1];
-
-      if (matches[2]) {
-        skin = parseInt(matches[2]);
-      }
-    }
-
-    if (Object.prototype.hasOwnProperty.call(data.short_names, emoji)) {
-      emoji = data.short_names[emoji];
-    }
-
-    if (Object.prototype.hasOwnProperty.call(data.emojis, emoji)) {
-      emojiData = data.emojis[emoji];
-    }
-  } else if (emoji.id) {
-    if (Object.prototype.hasOwnProperty.call(data.short_names, emoji.id)) {
-      emoji.id = data.short_names[emoji.id];
-    }
-
-    if (Object.prototype.hasOwnProperty.call(data.emojis, emoji.id)) {
-      emojiData = data.emojis[emoji.id];
-      skin = skin || emoji.skin;
-    }
-  }
-
-  if (!Object.keys(emojiData).length) {
-    emojiData = emoji;
-    emojiData.custom = true;
-
-    if (!emojiData.search) {
-      emojiData.search = buildSearch(emoji);
-    }
-  }
-
-  emojiData.emoticons = emojiData.emoticons || [];
-  emojiData.variations = emojiData.variations || [];
-
-  if (emojiData.skin_variations && skin > 1 && set) {
-    emojiData = JSON.parse(_JSON.stringify(emojiData));
-
-    const skinKey = SKINS[skin - 1],
-      variationData = emojiData.skin_variations[skinKey];
-
-    if (!variationData.variations && emojiData.variations) {
-      delete emojiData.variations;
-    }
-
-    if (variationData[`has_img_${set}`]) {
-      emojiData.skin_tone = skin;
-
-      for (const k in variationData) {
-        const v = variationData[k];
-        emojiData[k] = v;
-      }
-    }
-  }
-
-  if (emojiData.variations && emojiData.variations.length) {
-    emojiData = JSON.parse(_JSON.stringify(emojiData));
-    emojiData.unified = emojiData.variations.shift();
-  }
-
-  return emojiData;
-}
-
-function uniq(arr) {
-  return arr.reduce((acc, item) => {
-    if (acc.indexOf(item) === -1) {
-      acc.push(item);
-    }
-    return acc;
-  }, []);
-}
-
-function intersect(a, b) {
-  const uniqA = uniq(a);
-  const uniqB = uniq(b);
-
-  return uniqA.filter(item => uniqB.indexOf(item) >= 0);
-}
-
-function deepMerge(a, b) {
-  const o = {};
-
-  for (const key in a) {
-    const originalValue = a[key];
-    let value = originalValue;
-
-    if (Object.prototype.hasOwnProperty.call(b, key)) {
-      value = b[key];
-    }
-
-    if (typeof value === 'object') {
-      value = deepMerge(originalValue, value);
-    }
-
-    o[key] = value;
-  }
-
-  return o;
-}
-
-// https://github.com/sonicdoe/measure-scrollbar
-function measureScrollbar() {
-  const div = document.createElement('div');
-
-  div.style.width = '100px';
-  div.style.height = '100px';
-  div.style.overflow = 'scroll';
-  div.style.position = 'absolute';
-  div.style.top = '-9999px';
-
-  document.body.appendChild(div);
-  const scrollbarWidth = div.offsetWidth - div.clientWidth;
-  document.body.removeChild(div);
-
-  return scrollbarWidth;
-}
-
-export {
-  getData,
-  getSanitizedData,
-  uniq,
-  intersect,
-  deepMerge,
-  unifiedToNative,
-  measureScrollbar,
-};

app/soapbox/features/emoji/index.ts

@@ -1,41 +1,27 @@
-import data from '@emoji-mart/data';
+// import data from '@emoji-mart/data';
 import { load as cheerioLoad } from 'cheerio';
 import { parseDocument } from 'htmlparser2';
 
-import type EmojiData from '@emoji-mart/data';
+import unicodeMapping from './mapping';
+
 import type { Node as CheerioNode } from 'cheerio';
+import type { Emoji as EmojiMartEmoji } from 'emoji-mart';
 
-interface IUniMap {
+// export interface Emoji {
-  [s: string]: {
+//   id: string,
-    unified: string,
+//   custom: boolean,
-    shortcode: string,
+//   imageUrl: string,
-  }
+//   native: string,
-}
+//   colons: string,
+// }
 
-const generateMappings = (data: typeof EmojiData): IUniMap => {
+export type Emoji = any;
-  const result = {};
-  const emojis = Object.values(data.emojis ?? {});
-
-  for (const value of emojis) {
-    // @ts-ignore
-    for (const item of value.skins) {
-      const { unified, native } = item;
-
-      // @ts-ignore
-      result[native] = { unified, shortcode: value.id };
-    }
-  }
-
-  return result;
-};
-
-const unicodeMapping = generateMappings(data);
 
 const isAlphaNumeric = (c: string) => {
   const code = c.charCodeAt(0);
 
-  if (!(code > 47 && code < 58) && // numeric (0-9)
+  if (!(code > 47 && code < 58) &&  // numeric (0-9)
-      !(code > 64 && code < 91) && // upper alpha (A-Z)
+      !(code > 64 && code < 91) &&  // upper alpha (A-Z)
       !(code > 96 && code < 123)) { // lower alpha (a-z)
     return false;
   } else {
@@ -59,11 +45,11 @@ const convertUnicode = (c: string) => {
   return `<img draggable="false" class="emojione" alt="${c}" title=":${shortcode}:" src="/packs/emoji/${unified}.svg">`;
 };
 
-const convertEmoji = (str: string, customEmojis: any, autoplay: boolean) => {
+const convertEmoji = (str: string, customEmojis: any) => {
   if (str.length < 3) return str;
   if (str in customEmojis) {
     const emoji = customEmojis[str];
-    const filename = autoplay ? emoji.url : emoji.static_url;
+    const filename = emoji.static_url;
 
     if (filename?.length > 0) {
       return convertCustom(str, filename);
@@ -73,29 +59,33 @@ const convertEmoji = (str: string, customEmojis: any, autoplay: boolean) => {
   return str;
 };
 
-const popStack = (stack: string, open: boolean, res: string) => {
+const popStack = (stack: string, open: boolean) => {
-  res += stack;
+  const ret = stack;
   open = false;
   stack = '';
+  return ret;
 };
 
-const transmogrify = (str: string, customEmojis = {}, autoplay: boolean) => {
+// TODO: handle grouped unicode emojis
+export const emojifyText = (str: string, customEmojis = {}) => {
   let res = '';
   let stack = '';
   let open = false;
 
-  for (const c of Array.from(str)) { // Array.from respects unicode
+  for (const c of Array.from(str)) { // chunk by unicode codepoint with Array.from
     if (c in unicodeMapping) {
       if (open) { // unicode emoji inside colon
-        popStack(stack, open, res);
+        res += popStack(stack, open);
       }
 
       res += convertUnicode(c);
+
     } else if (c === ':') {
       stack += ':';
 
+      // we see another : we convert it and clear the stack buffer
       if (open) {
-        res += convertEmoji(stack, customEmojis, autoplay);
+        res += convertEmoji(stack, customEmojis);
         stack = '';
       }
 
@@ -104,8 +94,10 @@ const transmogrify = (str: string, customEmojis = {}, autoplay: boolean) => {
       if (open) {
         stack += c;
 
+        // if the stack is non-null and we see invalid chars it's a string not emoji
+        // so we push it to the return result and clear it
         if (!validEmojiChar(c)) {
-          popStack(stack, open, res);
+          res += popStack(stack, open);
         }
       } else {
         res += c;
@@ -113,14 +105,26 @@ const transmogrify = (str: string, customEmojis = {}, autoplay: boolean) => {
     }
   }
 
+  // never found a closing colon so it's just a raw string
+  if (open) {
+    res += stack;
+  }
+
   return res;
 };
 
+// const parseHmtl = (str: string) => {
+//   const ret = [];
+//   let depth = 0;
+//
+//   return ret;
+// }
+
 const filterTextNodes = (idx: number, el: CheerioNode) => {
   return el.nodeType === Node.TEXT_NODE;
 };
 
-const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
+const emojify = (str: string, customEmojis = {}) => {
   const dom = parseDocument(str);
   const $ = cheerioLoad(dom, {
     xmlMode: true,
@@ -128,8 +132,8 @@ const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
   });
 
   $.root()
-    .contents()
+    .contents() // iterate over flat map of all html elements
-    .filter(filterTextNodes)
+    .filter(filterTextNodes) // only iterate over text nodes
     .each((idx, el) => {
       // skip common case
       // @ts-ignore
@@ -137,7 +141,7 @@ const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
 
       // mutating el.data is incorrect but we do it to prevent a second dom parse
       // @ts-ignore
-      el.data = transmogrify(el.data, customEmojis, autoplay);
+      el.data = emojifyText(el.data, customEmojis);
     });
 
   return $.html();
@@ -145,12 +149,12 @@ const emojify = (str: string | any, customEmojis = {}, autoplay = false) => {
 
 export default emojify;
 
-export const buildCustomEmojis = (customEmojis: any, autoplay = false) => {
+export const buildCustomEmojis = (customEmojis: any) => {
-  const emojis: any[] = [];
+  const emojis: EmojiMartEmoji[] = [];
 
   customEmojis.forEach((emoji: any) => {
     const shortcode = emoji.get('shortcode');
-    const url       = autoplay ? emoji.get('url') : emoji.get('static_url');
+    const url       = emoji.get('static_url');
     const name      = shortcode.replace(':', '');
 
     emojis.push({

app/soapbox/features/emoji/mapping.ts

@@ -0,0 +1,31 @@
+import emojiData from './data';
+
+import type EmojiData from '@emoji-mart/data';
+
+interface IUniMap {
+  [s: string]: {
+    unified: string,
+    shortcode: string,
+  }
+}
+
+export const generateMappings = (data: typeof EmojiData): IUniMap => {
+  const result = {};
+  const emojis = Object.values(data.emojis ?? {});
+
+  for (const value of emojis) {
+    // @ts-ignore
+    for (const item of value.skins) {
+      const { unified, native } = item;
+
+      // @ts-ignore
+      result[native] = { unified, shortcode: value.id };
+    }
+  }
+
+  return result;
+};
+
+const unicodeMapping = generateMappings(emojiData);
+
+export default unicodeMapping;

app/soapbox/features/emoji/search.ts

@@ -0,0 +1,17 @@
+export interface searchOptions {
+  maxResults?: number;
+}
+
+export interface Emoji {
+
+}
+
+export const addCustomToPool = (customEmojis: Emoji[]) => {
+};
+
+const search = (str: string, options: searchOptions) => {
+  console.log(str, options);
+  return [];
+};
+
+export default search;

app/soapbox/features/emoji/unicode_to_filename.js

@@ -1,26 +0,0 @@
-// taken from:
-// https://github.com/twitter/twemoji/blob/47732c7/twemoji-generator.js#L848-L866
-exports.unicodeToFilename = (str) => {
-  let result = '';
-  let charCode = 0;
-  let p = 0;
-  let i = 0;
-  while (i < str.length) {
-    charCode = str.charCodeAt(i++);
-    if (p) {
-      if (result.length > 0) {
-        result += '-';
-      }
-      result += (0x10000 + ((p - 0xD800) << 10) + (charCode - 0xDC00)).toString(16);
-      p = 0;
-    } else if (0xD800 <= charCode && charCode <= 0xDBFF) {
-      p = charCode;
-    } else {
-      if (result.length > 0) {
-        result += '-';
-      }
-      result += charCode.toString(16);
-    }
-  }
-  return result;
-};

app/soapbox/features/emoji/unicode_to_unified_name.js

@@ -1,21 +0,0 @@
-function padLeft(str, num) {
-  while (str.length < num) {
-    str = '0' + str;
-  }
-
-  return str;
-}
-
-exports.unicodeToUnifiedName = (str) => {
-  let output = '';
-
-  for (let i = 0; i < str.length; i += 2) {
-    if (i > 0) {
-      output += '-';
-    }
-
-    output += padLeft(str.codePointAt(i).toString(16).toUpperCase(), 4);
-  }
-
-  return output;
-};

app/soapbox/features/ui/components/link_footer.tsx

@@ -6,7 +6,7 @@ import { Link } from 'react-router-dom';
 
 import { logOut } from 'soapbox/actions/auth';
 import { Text } from 'soapbox/components/ui';
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { useSoapboxConfig, useOwnAccount, useFeatures } from 'soapbox/hooks';
 import sourceCode from 'soapbox/utils/code';
 

app/soapbox/normalizers/account.ts

@@ -11,7 +11,7 @@ import {
   fromJS,
 } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeEmoji } from 'soapbox/normalizers/emoji';
 import { unescapeHTML } from 'soapbox/utils/html';
 import { mergeDefined, makeEmojiMap } from 'soapbox/utils/normalizers';

app/soapbox/normalizers/poll.ts

@@ -11,7 +11,7 @@ import {
   fromJS,
 } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeEmoji } from 'soapbox/normalizers/emoji';
 import { makeEmojiMap } from 'soapbox/utils/normalizers';
 

app/soapbox/normalizers/status_edit.ts

@@ -9,7 +9,7 @@ import {
   fromJS,
 } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeAttachment } from 'soapbox/normalizers/attachment';
 import { normalizeEmoji } from 'soapbox/normalizers/emoji';
 import { normalizePoll } from 'soapbox/normalizers/poll';

app/soapbox/reducers/compose.ts

@@ -59,7 +59,7 @@ import { normalizeAttachment } from '../normalizers/attachment';
 import { unescapeHTML } from '../utils/html';
 
 import type { AnyAction } from 'redux';
-import type { Emoji } from 'soapbox/components/autosuggest_emoji';
+import type { Emoji } from 'soapbox/features/emoji';
 import type {
   Account as AccountEntity,
   APIEntity,

app/soapbox/reducers/custom_emojis.ts

@@ -1,10 +1,10 @@
 import { List as ImmutableList, Map as ImmutableMap, fromJS } from 'immutable';
 
-import { emojis as emojiData } from 'soapbox/features/emoji/emoji_mart_data_light';
+import { buildCustomEmojis } from 'soapbox/features/emoji';
-import { addCustomToPool } from 'soapbox/features/emoji/emoji_mart_search_light';
+import { addCustomToPool } from 'soapbox/features/emoji/search';
+// import emojiData from 'soapbox/features/emoji/data';
 
 import { CUSTOM_EMOJIS_FETCH_SUCCESS } from '../actions/custom_emojis';
-import { buildCustomEmojis } from '../features/emoji/emoji';
 
 import type { AnyAction } from 'redux';
 import type { APIEntity } from 'soapbox/types/entities';
@@ -17,14 +17,18 @@ const autosuggestPopulate = (emojis: ImmutableList<ImmutableMap<string, string>>
 };
 
 const importEmojis = (customEmojis: APIEntity[]) => {
-  const emojis = (fromJS(customEmojis) as ImmutableList<ImmutableMap<string, string>>).filter((emoji) => {
+  // const emojis = (fromJS(customEmojis)).filter((emoji) => {
-    // If a custom emoji has the shortcode of a Unicode emoji, skip it.
+  //   // If a custom emoji has the shortcode of a Unicode emoji, skip it.
-    // Otherwise it breaks EmojiMart.
+  //   // Otherwise it breaks EmojiMart.
-    // https://gitlab.com/soapbox-pub/soapbox-fe/-/issues/610
+  //   // https://gitlab.com/soapbox-pub/soapbox-fe/-/issues/610
-    const shortcode = emoji.get('shortcode', '').toLowerCase();
+  //   const shortcode = emoji.get('shortcode', '').toLowerCase();
-    return !emojiData[shortcode];
+  //   return !emojiData.emojis[shortcode];
-  });
+  // });
 
+  // @ts-ignore
+  const emojis = fromJS(customEmojis);
+
+  // @ts-ignore
   autosuggestPopulate(emojis);
   return emojis;
 };

app/soapbox/reducers/statuses.ts

@@ -1,7 +1,7 @@
 import escapeTextContentForBrowser from 'escape-html';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 
-import emojify from 'soapbox/features/emoji/emoji';
+import emojify from 'soapbox/features/emoji';
 import { normalizeStatus } from 'soapbox/normalizers';
 import { simulateEmojiReact, simulateUnEmojiReact } from 'soapbox/utils/emoji_reacts';
 import { stripCompatibilityFeatures, unescapeHTML } from 'soapbox/utils/html';

package.json

@@ -116,7 +116,7 @@
     "emoji-datasource": "5.0.0",
     "emoji-mart": "^5.1.0",
     "emoji-mart-old": "npm:emoji-mart-lazyload",
-    "entities": "^3.0.1",
+    "entities": "^4.3.1",
     "es6-symbol": "^3.1.1",
     "escape-html": "^1.0.3",
     "exif-js": "^2.3.0",

types/emoji-mart/index.d.ts

@@ -1,6 +1,17 @@
 declare module 'emoji-mart' {
-  export type PickerProps = {
+  export interface EmojiSkin {
-    custom?: { emojis: any[] }[],
+    src: string
+  }
+
+  export interface Emoji {
+    id: string,
+    name: string,
+    keywords: string[],
+    skins: EmojiSkin[],
+  }
+
+  export interface PickerProps {
+    custom?: { emojis: Emoji[] }[],
     set?: string,
     title?: string,
     theme?: string,

yarn.lock

@@ -5184,12 +5184,7 @@ entities@^2.0.0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
   integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
 
-entities@^3.0.1:
+entities@^4.2.0, entities@^4.3.0, entities@^4.3.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/entities/-/entities-3.0.1.tgz#2b887ca62585e96db3903482d336c1006c3001d4"
-  integrity sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q==
-
-entities@^4.2.0, entities@^4.3.0:
   version "4.3.1"
   resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.1.tgz#c34062a94c865c322f9d67b4384e4169bcede6a4"
   integrity sha512-o4q/dYJlmyjP2zfnaWDUC6A3BQFmVTX+tZPezK7k0GLSU9QYCauscf5Y+qcEPzKL+EixVouYDgLQK5H9GrLpkg==