diff --git a/packages/pl-fe/package.json b/packages/pl-fe/package.json index 75262306a0..28837b1f94 100644 --- a/packages/pl-fe/package.json +++ b/packages/pl-fe/package.json @@ -132,7 +132,6 @@ "reselect": "^5.1.1", "resize-observer-polyfill": "^1.5.1", "sass": "^1.79.4", - "semver": "^7.6.3", "stringz": "^2.1.0", "tiny-queue": "^0.2.1", "tslib": "^2.7.0", diff --git a/packages/pl-fe/src/features/emoji/index.ts b/packages/pl-fe/src/features/emoji/index.ts index 1e85fc74f3..aafe331d6a 100644 --- a/packages/pl-fe/src/features/emoji/index.ts +++ b/packages/pl-fe/src/features/emoji/index.ts @@ -2,8 +2,8 @@ import split from 'graphemesplit'; import unicodeMapping from './mapping'; +import type { Emoji as EmojiMart, CustomEmoji as EmojiMartCustom } from './data'; import type { CustomEmoji as BaseCustomEmoji } from 'pl-api'; -import type { Emoji as EmojiMart, CustomEmoji as EmojiMartCustom } from 'pl-fe/features/emoji/data'; /* * TODO: Consolate emoji object types diff --git a/packages/pl-fe/src/features/emoji/mapping-compiletime.ts b/packages/pl-fe/src/features/emoji/mapping-compiletime.ts new file mode 100644 index 0000000000..f5ef8cdf0a --- /dev/null +++ b/packages/pl-fe/src/features/emoji/mapping-compiletime.ts @@ -0,0 +1,107 @@ +import { createRequire } from 'node:module'; + +import type { EmojiData } from './data'; +import type { UnicodeMap } from './mapping'; + +const require = createRequire(import.meta.url); +const data = require('@emoji-mart/data/sets/14/twitter.json'); + +const stripLeadingZeros = /^0+/; + +/* + * Twemoji strips their hex codes from unicode codepoints to make it look "pretty" + * - leading 0s are removed + * - fe0f is removed unless it has 200d + * - fe0f is NOT removed for 1f441-fe0f-200d-1f5e8-fe0f even though it has a 200d + * + * this is all wrong + */ +const blacklist = { + '1f441-fe0f-200d-1f5e8-fe0f': true, +}; + +const tweaks = { + '#⃣': ['23-20e3', 'hash'], + '*⃣': ['2a-20e3', 'keycap_star'], + '0⃣': ['30-20e3', 'zero'], + '1⃣': ['31-20e3', 'one'], + '2⃣': ['32-20e3', 'two'], + '3⃣': ['33-20e3', 'three'], + '4⃣': ['34-20e3', 'four'], + '5⃣': ['35-20e3', 'five'], + '6⃣': ['36-20e3', 'six'], + '7⃣': ['37-20e3', 'seven'], + '8⃣': ['38-20e3', 'eight'], + '9⃣': ['39-20e3', 'nine'], + '❤‍🔥': ['2764-fe0f-200d-1f525', 'heart_on_fire'], + '❤‍🩹': ['2764-fe0f-200d-1fa79', 'mending_heart'], + '👁‍🗨️': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], + '👁️‍🗨': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], + '👁‍🗨': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], + '🕵‍♂️': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], + '🕵️‍♂': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], + '🕵‍♂': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], + '🕵‍♀️': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], + '🕵️‍♀': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], + '🕵‍♀': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], + '🏌‍♂️': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], + '🏌️‍♂': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], + '🏌‍♂': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], + '🏌‍♀️': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], + '🏌️‍♀': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], + '🏌‍♀': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], + '⛹‍♂️': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], + '⛹️‍♂': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], + '⛹‍♂': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], + '⛹‍♀️': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], + '⛹️‍♀': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], + '⛹‍♀': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], + '🏋‍♂️': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], + '🏋️‍♂': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], + '🏋‍♂': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], + '🏋‍♀️': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], + '🏋️‍♀': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], + '🏋‍♀': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], + '🏳‍🌈': ['1f3f3-fe0f-200d-1f308', 'rainbow_flag'], + '🏳‍⚧️': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], + '🏳️‍⚧': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], + '🏳‍⚧': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], +}; + +const stripcodes = (unified: string, native: string) => { + const stripped = unified.replace(stripLeadingZeros, ''); + + if (unified.includes('200d') && !(unified in blacklist)) { + return stripped; + } else { + return stripped.replaceAll('-fe0f', ''); + } +}; + +const generateMappings = (emojiMap: EmojiData['emojis']): UnicodeMap => { + const result: UnicodeMap = {}; + const emojis = Object.values(emojiMap ?? {}); + + for (const value of emojis) { + for (const item of value.skins) { + const { unified, native } = item; + const stripped = stripcodes(unified, native); + + result[native] = { unified: stripped, shortcode: value.id }; + } + } + + for (const [native, [unified, shortcode]] of Object.entries(tweaks)) { + const stripped = stripcodes(unified, native); + + result[native] = { unified: stripped, shortcode }; + } + + return result; +}; + +const unicodeMapping = generateMappings(data.emojis); + +export default () => ({ + data: unicodeMapping, +}); diff --git a/packages/pl-fe/src/features/emoji/mapping.ts b/packages/pl-fe/src/features/emoji/mapping.ts index 5d5ba507aa..a76d52245f 100644 --- a/packages/pl-fe/src/features/emoji/mapping.ts +++ b/packages/pl-fe/src/features/emoji/mapping.ts @@ -1,6 +1,3 @@ -import data, { EmojiData } from './data'; - -const stripLeadingZeros = /^0+/; interface UnicodeMap { [s: string]: { unified: string; @@ -8,99 +5,6 @@ interface UnicodeMap { }; } -/* - * Twemoji strips their hex codes from unicode codepoints to make it look "pretty" - * - leading 0s are removed - * - fe0f is removed unless it has 200d - * - fe0f is NOT removed for 1f441-fe0f-200d-1f5e8-fe0f even though it has a 200d - * - * this is all wrong - */ +export default import.meta.compileTime('./mapping-compiletime.ts'); -const blacklist = { - '1f441-fe0f-200d-1f5e8-fe0f': true, -}; - -const tweaks = { - '#⃣': ['23-20e3', 'hash'], - '*⃣': ['2a-20e3', 'keycap_star'], - '0⃣': ['30-20e3', 'zero'], - '1⃣': ['31-20e3', 'one'], - '2⃣': ['32-20e3', 'two'], - '3⃣': ['33-20e3', 'three'], - '4⃣': ['34-20e3', 'four'], - '5⃣': ['35-20e3', 'five'], - '6⃣': ['36-20e3', 'six'], - '7⃣': ['37-20e3', 'seven'], - '8⃣': ['38-20e3', 'eight'], - '9⃣': ['39-20e3', 'nine'], - '❤‍🔥': ['2764-fe0f-200d-1f525', 'heart_on_fire'], - '❤‍🩹': ['2764-fe0f-200d-1fa79', 'mending_heart'], - '👁‍🗨️': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], - '👁️‍🗨': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], - '👁‍🗨': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], - '🕵‍♂️': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], - '🕵️‍♂': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], - '🕵‍♂': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], - '🕵‍♀️': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], - '🕵️‍♀': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], - '🕵‍♀': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], - '🏌‍♂️': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], - '🏌️‍♂': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], - '🏌‍♂': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], - '🏌‍♀️': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], - '🏌️‍♀': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], - '🏌‍♀': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], - '⛹‍♂️': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], - '⛹️‍♂': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], - '⛹‍♂': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], - '⛹‍♀️': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], - '⛹️‍♀': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], - '⛹‍♀': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], - '🏋‍♂️': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], - '🏋️‍♂': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], - '🏋‍♂': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], - '🏋‍♀️': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], - '🏋️‍♀': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], - '🏋‍♀': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], - '🏳‍🌈': ['1f3f3-fe0f-200d-1f308', 'rainbow_flag'], - '🏳‍⚧️': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], - '🏳️‍⚧': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], - '🏳‍⚧': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], -}; - -const stripcodes = (unified: string, native: string) => { - const stripped = unified.replace(stripLeadingZeros, ''); - - if (unified.includes('200d') && !(unified in blacklist)) { - return stripped; - } else { - return stripped.replaceAll('-fe0f', ''); - } -}; - -const generateMappings = (data: EmojiData): UnicodeMap => { - const result: UnicodeMap = {}; - const emojis = Object.values(data.emojis ?? {}); - - for (const value of emojis) { - for (const item of value.skins) { - const { unified, native } = item; - const stripped = stripcodes(unified, native); - - result[native] = { unified: stripped, shortcode: value.id }; - } - } - - for (const [native, [unified, shortcode]] of Object.entries(tweaks)) { - const stripped = stripcodes(unified, native); - - result[native] = { unified: stripped, shortcode }; - } - - return result; -}; - -const unicodeMapping = generateMappings(data); - -export { generateMappings, unicodeMapping as default }; +export type { UnicodeMap }; diff --git a/packages/pl-fe/src/features/emoji/search.ts b/packages/pl-fe/src/features/emoji/search.ts index 0316f6445d..286873ca75 100644 --- a/packages/pl-fe/src/features/emoji/search.ts +++ b/packages/pl-fe/src/features/emoji/search.ts @@ -1,21 +1,26 @@ import FlexSearch from 'flexsearch'; -import data from './data'; - +import type { EmojiData } from './data'; import type { Emoji } from './index'; import type { CustomEmoji } from 'pl-api'; +let emojis: EmojiData['emojis'] = {}; + +import('./data').then(data => { + emojis = data.emojis; + + const sortedEmojis = Object.entries(emojis).sort((a, b) => a[0].localeCompare(b[0])); + for (const [key, emoji] of sortedEmojis) { + index.add('n' + key, `${emoji.id} ${emoji.name} ${emoji.keywords.join(' ')}`); + } +}).catch(() => { }); + const index = new FlexSearch.Index({ tokenize: 'full', optimize: true, context: true, }); -const sortedEmojis = Object.entries(data.emojis).sort((a, b) => a[0].localeCompare(b[0])); -for (const [key, emoji] of sortedEmojis) { - index.add('n' + key, `${emoji.id} ${emoji.name} ${emoji.keywords.join(' ')}`); -} - interface searchOptions { maxResults?: number; custom?: any; @@ -58,7 +63,7 @@ const search = ( } } - const skins = data.emojis[id.slice(1)]?.skins; + const skins = emojis[id.slice(1)]?.skins; if (skins) { return { diff --git a/packages/pl-fe/src/reducers/custom-emojis.ts b/packages/pl-fe/src/reducers/custom-emojis.ts index d6fe9f49e3..4273d44c90 100644 --- a/packages/pl-fe/src/reducers/custom-emojis.ts +++ b/packages/pl-fe/src/reducers/custom-emojis.ts @@ -1,5 +1,4 @@ import { buildCustomEmojis } from 'pl-fe/features/emoji'; -import emojiData from 'pl-fe/features/emoji/data'; import { addCustomToPool } from 'pl-fe/features/emoji/search'; import { CUSTOM_EMOJIS_FETCH_SUCCESS, type CustomEmojisAction } from '../actions/custom-emojis'; @@ -8,27 +7,10 @@ import type { CustomEmoji } from 'pl-api'; const initialState: Array = []; -// Populate custom emojis for composer autosuggest -const autosuggestPopulate = (emojis: Array) => { - addCustomToPool(buildCustomEmojis(emojis)); -}; - -const importEmojis = (customEmojis: Array) => { - const emojis = customEmojis.filter((emoji) => { - // If a custom emoji has the shortcode of a Unicode emoji, skip it. - // Otherwise it breaks EmojiMart. - // https://gitlab.com/soapbox-pub/soapbox/-/issues/610 - const shortcode = emoji.shortcode.toLowerCase(); - return !emojiData.emojis[shortcode]; - }); - - autosuggestPopulate(emojis); - return emojis; -}; - const custom_emojis = (state = initialState, action: CustomEmojisAction) => { if (action.type === CUSTOM_EMOJIS_FETCH_SUCCESS) { - return importEmojis(action.custom_emojis); + addCustomToPool(buildCustomEmojis(action.custom_emojis)); + return action.custom_emojis; } return state;