Merge branch 'more-entity-normalization' into 'develop'

More entity normalization

See merge request soapbox-pub/soapbox-fe!1091

app/soapbox/__fixtures__/pleroma-status-with-poll-with-emojis.json

@@ -0,0 +1,236 @@
+{
+  "account": {
+    "acct": "alex",
+    "avatar": "https://media.gleasonator.com/6d64aecb17348b23aaff78db4687b9476cb0da1c07cc6a819c2e6ec7144c18b1.png",
+    "avatar_static": "https://media.gleasonator.com/6d64aecb17348b23aaff78db4687b9476cb0da1c07cc6a819c2e6ec7144c18b1.png",
+    "bot": false,
+    "created_at": "2020-01-08T01:25:43.000Z",
+    "display_name": "Alex Gleason",
+    "emojis": [],
+    "fields": [
+      {
+        "name": "Website",
+        "value": "<a href=\"https://alexgleason.me\" rel=\"ugc\">https://alexgleason.me</a>"
+      },
+      {
+        "name": "Soapbox",
+        "value": "<a href=\"https://soapbox.pub\" rel=\"ugc\">https://soapbox.pub</a>"
+      },
+      {
+        "name": "Email",
+        "value": "alex@alexgleason.me"
+      },
+      {
+        "name": "Gender identity",
+        "value": "Soyboy"
+      },
+      {
+        "name": "Donate (PayPal)",
+        "value": "<a href=\"https://paypal.me/gleasonator\" rel=\"ugc\">https://paypal.me/gleasonator</a>"
+      },
+      {
+        "name": "$BTC",
+        "value": "bc1q9cx35adpm73aq2fw40ye6ts8hfxqzjr5unwg0n"
+      },
+      {
+        "name": "$ETH",
+        "value": "0xAc9aB5Fc04Dc1cB1789Af75b523Bd23C70B2D717"
+      },
+      {
+        "name": "$DOGE",
+        "value": "D5zVZs6jrRakaPVGiErkQiHt9sayzm6V5D"
+      },
+      {
+        "name": "$XMR",
+        "value": "45JDCLrjJ4bgVUSbbs2yjy9m5Mf4VLPW8fG7jw9sq5u69rXZZopQogZNeyYkMBnXpkaip4p4QwaaJNhdTotPa9g44DBCzdK"
+      }
+    ],
+    "followers_count": 2467,
+    "following_count": 1581,
+    "fqn": "alex@gleasonator.com",
+    "header": "https://media.gleasonator.com/accounts/headers/000/000/001/original/9d0e4dbf1c9dbc8f.png",
+    "header_static": "https://media.gleasonator.com/accounts/headers/000/000/001/original/9d0e4dbf1c9dbc8f.png",
+    "id": "9v5bmRalQvjOy0ECcC",
+    "last_status_at": "2022-03-11T01:33:19",
+    "locked": false,
+    "note": "I create Fediverse software that empowers people online.<br/><br/>I&#39;m vegan btw<br/><br/>Note: If you have a question for me, please tag me publicly. This gives the opportunity for others to chime in, and bystanders to learn.",
+    "pleroma": {
+      "accepts_chat_messages": true,
+      "also_known_as": [
+        "https://mitra.social/users/alex"
+      ],
+      "ap_id": "https://gleasonator.com/users/alex",
+      "background_image": null,
+      "birthday": "1993-07-03",
+      "favicon": "https://gleasonator.com/favicon.png",
+      "hide_favorites": true,
+      "hide_followers": false,
+      "hide_followers_count": false,
+      "hide_follows": false,
+      "hide_follows_count": false,
+      "is_admin": true,
+      "is_confirmed": true,
+      "is_moderator": false,
+      "is_suggested": true,
+      "relationship": {},
+      "skip_thread_containment": false,
+      "tags": []
+    },
+    "source": {
+      "fields": [
+        {
+          "name": "Website",
+          "value": "https://alexgleason.me"
+        },
+        {
+          "name": "Soapbox",
+          "value": "https://soapbox.pub"
+        },
+        {
+          "name": "Email",
+          "value": "alex@alexgleason.me"
+        },
+        {
+          "name": "Gender identity",
+          "value": "Soyboy"
+        },
+        {
+          "name": "Donate (PayPal)",
+          "value": "https://paypal.me/gleasonator"
+        },
+        {
+          "name": "$BTC",
+          "value": "bc1q9cx35adpm73aq2fw40ye6ts8hfxqzjr5unwg0n"
+        },
+        {
+          "name": "$ETH",
+          "value": "0xAc9aB5Fc04Dc1cB1789Af75b523Bd23C70B2D717"
+        },
+        {
+          "name": "$DOGE",
+          "value": "D5zVZs6jrRakaPVGiErkQiHt9sayzm6V5D"
+        },
+        {
+          "name": "$XMR",
+          "value": "45JDCLrjJ4bgVUSbbs2yjy9m5Mf4VLPW8fG7jw9sq5u69rXZZopQogZNeyYkMBnXpkaip4p4QwaaJNhdTotPa9g44DBCzdK"
+        }
+      ],
+      "note": "I create Fediverse software that empowers people online.\r\n\r\nI'm vegan btw\r\n\r\nNote: If you have a question for me, please tag me publicly. This gives the opportunity for others to chime in, and bystanders to learn.",
+      "pleroma": {
+        "actor_type": "Person",
+        "discoverable": false
+      },
+      "sensitive": false
+    },
+    "statuses_count": 23651,
+    "url": "https://gleasonator.com/users/alex",
+    "username": "alex"
+  },
+  "application": {
+    "name": "Soapbox FE",
+    "website": "https://soapbox.pub/"
+  },
+  "bookmarked": false,
+  "card": null,
+  "content": "<p>Test poll</p>",
+  "created_at": "2022-03-11T01:33:18.000Z",
+  "emojis": [
+    {
+      "shortcode": "gleason_excited",
+      "static_url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+      "url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+      "visible_in_picker": false
+    },
+    {
+      "shortcode": "soapbox",
+      "static_url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+      "url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+      "visible_in_picker": false
+    }
+  ],
+  "favourited": false,
+  "favourites_count": 1,
+  "id": "AHHue68kB59xtUv7MO",
+  "in_reply_to_account_id": null,
+  "in_reply_to_id": null,
+  "language": null,
+  "media_attachments": [],
+  "mentions": [],
+  "muted": false,
+  "pinned": false,
+  "pleroma": {
+    "content": {
+      "text/plain": "Test poll"
+    },
+    "conversation_id": "AHHue65YMwbjjbQZO4",
+    "direct_conversation_id": null,
+    "emoji_reactions": [],
+    "expires_at": null,
+    "in_reply_to_account_acct": null,
+    "local": true,
+    "parent_visible": false,
+    "pinned_at": null,
+    "quote": null,
+    "quote_url": null,
+    "quote_visible": false,
+    "spoiler_text": {
+      "text/plain": ""
+    },
+    "thread_muted": false
+  },
+  "poll": {
+    "emojis": [
+      {
+        "shortcode": "gleason_excited",
+        "static_url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+        "url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+        "visible_in_picker": false
+      },
+      {
+        "shortcode": "soapbox",
+        "static_url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+        "url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+        "visible_in_picker": false
+      }
+    ],
+    "expired": false,
+    "expires_at": "2022-03-12T01:33:18.000Z",
+    "id": "AHHue67gF2JDqCQGhc",
+    "multiple": false,
+    "options": [
+      {
+        "title": "Regular emoji 😍 ",
+        "votes_count": 0
+      },
+      {
+        "title": "Custom emoji :gleason_excited: ",
+        "votes_count": 1
+      },
+      {
+        "title": "No emoji",
+        "votes_count": 0
+      },
+      {
+        "title": "🤔 😮 😠 ",
+        "votes_count": 1
+      },
+      {
+        "title": ":soapbox:",
+        "votes_count": 1
+      }
+    ],
+    "voters_count": 3,
+    "votes_count": 3
+  },
+  "reblog": null,
+  "reblogged": false,
+  "reblogs_count": 1,
+  "replies_count": 1,
+  "sensitive": false,
+  "spoiler_text": "",
+  "tags": [],
+  "text": null,
+  "uri": "https://gleasonator.com/objects/46d2ab26-3497-442b-999f-612fe717b0a3",
+  "url": "https://gleasonator.com/notice/AHHue68kB59xtUv7MO",
+  "visibility": "public"
+}

app/soapbox/actions/importer/index.js

@@ -1,9 +1,6 @@
 import { getSettings } from '../settings';
 
-import {
+import { normalizeAccount } from './normalizer';
-  normalizeAccount,
-  normalizePoll,
-} from './normalizer';
 
 export const ACCOUNT_IMPORT  = 'ACCOUNT_IMPORT';
 export const ACCOUNTS_IMPORT = 'ACCOUNTS_IMPORT';
@@ -130,7 +127,7 @@ export function importFetchedStatuses(statuses) {
       }
 
       if (status.poll?.id) {
-        polls.push(normalizePoll(status.poll));
+        polls.push(status.poll);
       }
     }
 
@@ -144,7 +141,7 @@ export function importFetchedStatuses(statuses) {
 
 export function importFetchedPoll(poll) {
   return dispatch => {
-    dispatch(importPolls([normalizePoll(poll)]));
+    dispatch(importPolls([poll]));
   };
 }
 

app/soapbox/actions/importer/normalizer.js

@@ -41,20 +41,6 @@ export function normalizeAccount(account) {
   return account;
 }
 
-export function normalizePoll(poll) {
-  const normalPoll = { ...poll };
-
-  const emojiMap = makeEmojiMap(normalPoll);
-
-  normalPoll.options = poll.options.map((option, index) => ({
-    ...option,
-    voted: Boolean(poll.own_votes?.includes(index)),
-    title_emojified: emojify(escapeTextContentForBrowser(option.title), emojiMap),
-  }));
-
-  return normalPoll;
-}
-
 export function normalizeChat(chat, normalOldChat) {
   const normalChat   = { ...chat };
   const { account, last_message: lastMessage } = chat;

app/soapbox/components/poll.js

@@ -1,5 +1,4 @@
 import classNames from 'classnames';
-import escapeTextContentForBrowser from 'escape-html';
 import PropTypes from 'prop-types';
 import React from 'react';
 import ImmutablePropTypes from 'react-immutable-proptypes';
@@ -10,8 +9,8 @@ import spring from 'react-motion/lib/spring';
 import { openModal } from 'soapbox/actions/modals';
 import { vote, fetchPoll } from 'soapbox/actions/polls';
 import Icon from 'soapbox/components/icon';
-import emojify from 'soapbox/features/emoji/emoji';
 import Motion from 'soapbox/features/ui/util/optional_motion';
+import SoapboxPropTypes from 'soapbox/utils/soapbox_prop_types';
 
 import RelativeTimestamp from './relative_timestamp';
 
@@ -21,11 +20,6 @@ const messages = defineMessages({
   votes: { id: 'poll.votes', defaultMessage: '{votes, plural, one {# vote} other {# votes}}' },
 });
 
-const makeEmojiMap = record => record.get('emojis').reduce((obj, emoji) => {
-  obj[`:${emoji.get('shortcode')}:`] = emoji.toJS();
-  return obj;
-}, {});
-
 export default @injectIntl
 class Poll extends ImmutablePureComponent {
 
@@ -34,6 +28,7 @@ class Poll extends ImmutablePureComponent {
     intl: PropTypes.object.isRequired,
     dispatch: PropTypes.func,
     disabled: PropTypes.bool,
+    me: SoapboxPropTypes.me,
     status: PropTypes.string,
   };
 
@@ -104,12 +99,7 @@ class Poll extends ImmutablePureComponent {
     const leading = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') >= other.get('votes_count'));
     const active  = !!this.state.selected[`${optionIndex}`];
     const voted   = poll.own_votes?.includes(optionIndex);
-
+    const titleEmojified = option.get('title_emojified');
-    let titleEmojified = option.get('title_emojified');
-    if (!titleEmojified) {
-      const emojiMap = makeEmojiMap(poll);
-      titleEmojified = emojify(escapeTextContentForBrowser(option.get('title')), emojiMap);
-    }
 
     return (
       <li key={option.get('title')}>

app/soapbox/containers/poll_container.js

@@ -4,6 +4,7 @@ import Poll from 'soapbox/components/poll';
 
 const mapStateToProps = (state, { pollId }) => ({
   poll: state.getIn(['polls', pollId]),
+  me: state.get('me'),
 });
 
 

app/soapbox/normalizers/__tests__/status-test.js

@@ -32,71 +32,79 @@ describe('normalizeStatus', () => {
   it('adds mention to self in self-reply on Mastodon', () => {
     const status = fromJS(require('soapbox/__fixtures__/mastodon-reply-to-self.json'));
 
-    const expected = fromJS([{
+    const expected = {
       id: '106801667066418367',
       username: 'benis911',
       acct: 'benis911',
       url: 'https://mastodon.social/@benis911',
-    }]);
+    };
 
-    const result = normalizeStatus(status).get('mentions');
+    const result = normalizeStatus(status).mentions;
 
-    expect(result).toEqual(expected);
+    expect(result.size).toBe(1);
+    expect(result.get(0).toJS()).toMatchObject(expected);
+    expect(result.get(0).id).toEqual('106801667066418367');
+    expect(ImmutableRecord.isRecord(result.get(0))).toBe(true);
   });
 
   it('normalizes mentions with only acct', () => {
     const status = fromJS({ mentions: [{ acct: 'alex@gleasonator.com' }] });
 
-    const expected = fromJS([{
+    const expected = [{
+      id: '',
       acct: 'alex@gleasonator.com',
       username: 'alex',
       url: '',
-    }]);
+    }];
 
     const result = normalizeStatus(status).get('mentions');
 
-    expect(result).toEqual(expected);
+    expect(result.toJS()).toEqual(expected);
   });
 
   it('normalizes Mitra attachments', () => {
     const status = fromJS(require('soapbox/__fixtures__/mitra-status-with-attachments.json'));
 
-    const expected = fromJS([{
+    const expected = [{
       id: '017eeb0e-e5df-30a4-77a7-a929145cb836',
       type: 'image',
       url: 'https://mitra.social/media/8e04e6091bbbac79641b5812508683ce72c38693661c18d16040553f2371e18d.png',
       preview_url: 'https://mitra.social/media/8e04e6091bbbac79641b5812508683ce72c38693661c18d16040553f2371e18d.png',
-      remote_url: 'https://mitra.social/media/8e04e6091bbbac79641b5812508683ce72c38693661c18d16040553f2371e18d.png',
+      remote_url: null,
     }, {
       id: '017eeb0e-e5e4-2a48-2889-afdebf368a54',
       type: 'unknown',
       url: 'https://mitra.social/media/8f72dc2e98572eb4ba7c3a902bca5f69c448fc4391837e5f8f0d4556280440ac',
       preview_url: 'https://mitra.social/media/8f72dc2e98572eb4ba7c3a902bca5f69c448fc4391837e5f8f0d4556280440ac',
-      remote_url: 'https://mitra.social/media/8f72dc2e98572eb4ba7c3a902bca5f69c448fc4391837e5f8f0d4556280440ac',
+      remote_url: null,
     }, {
       id: '017eeb0e-e5e5-79fd-6054-8b6869b1db49',
       type: 'unknown',
       url: 'https://mitra.social/media/55a81a090247cc4fc127e5716bcf7964f6e0df9b584f85f4696c0b994747a4d0.oga',
       preview_url: 'https://mitra.social/media/55a81a090247cc4fc127e5716bcf7964f6e0df9b584f85f4696c0b994747a4d0.oga',
-      remote_url: 'https://mitra.social/media/55a81a090247cc4fc127e5716bcf7964f6e0df9b584f85f4696c0b994747a4d0.oga',
+      remote_url: null,
     }, {
       id: '017eeb0e-e5e6-c416-a444-21e560c47839',
       type: 'unknown',
       url: 'https://mitra.social/media/0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0',
       preview_url: 'https://mitra.social/media/0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0',
-      remote_url: 'https://mitra.social/media/0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0',
+      remote_url: null,
-    }]);
+    }];
 
     const result = normalizeStatus(status);
 
-    expect(result.media_attachments).toEqual(expected);
+    expect(result.media_attachments.toJS()).toMatchObject(expected);
   });
 
   it('leaves Pleroma attachments alone', () => {
     const status = fromJS(require('soapbox/__fixtures__/pleroma-status-with-attachments.json'));
-    const result = normalizeStatus(status);
+    const result = normalizeStatus(status).media_attachments;
 
-    expect(status.get('media_attachments')).toEqual(result.media_attachments);
+    expect(result.size).toBe(4);
+    expect(result.get(0).text_url).toBe(undefined);
+    expect(result.get(1).meta).toEqual(fromJS({}));
+    expect(result.getIn([1, 'pleroma', 'mime_type'])).toBe('application/x-nes-rom');
+    expect(ImmutableRecord.isRecord(result.get(3))).toBe(true);
   });
 
   it('normalizes Pleroma quote post', () => {
@@ -165,4 +173,17 @@ describe('normalizeStatus', () => {
     expect(result.poll.voted).toBe(false);
     expect(result.poll.own_votes).toBe(null);
   });
+
+  it('normalizes poll with emojis', () => {
+    const status = fromJS(require('soapbox/__fixtures__/pleroma-status-with-poll-with-emojis.json'));
+    const result = normalizeStatus(status);
+
+    // Emojifies poll options
+    expect(result.poll.options.get(1).title_emojified)
+      .toEqual('Custom emoji <img draggable="false" class="emojione" alt=":gleason_excited:" title=":gleason_excited:" src="https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png" /> ');
+
+    // Parses emojis as Immutable.Record's
+    expect(ImmutableRecord.isRecord(result.poll.emojis.get(0))).toBe(true);
+    expect(result.poll.emojis.get(1).shortcode).toEqual('soapbox');
+  });
 });

app/soapbox/normalizers/account.ts

@@ -97,6 +97,13 @@ const normalizeLocation = (account: ImmutableMap<string, any>) => {
   });
 };
 
+// Set username from acct, if applicable
+const fixUsername = (account: ImmutableMap<string, any>) => {
+  return account.update('username', username => (
+    username || (account.get('acct') || '').split('@')[0]
+  ));
+};
+
 export const normalizeAccount = (account: ImmutableMap<string, any>): IAccount => {
   return AccountRecord(
     account.withMutations(account => {
@@ -104,6 +111,7 @@ export const normalizeAccount = (account: ImmutableMap<string, any>): IAccount =
       normalizeVerified(account);
       normalizeBirthday(account);
       normalizeLocation(account);
+      fixUsername(account);
     }),
   );
 };

app/soapbox/normalizers/instance.js

@@ -94,9 +94,8 @@ export const normalizeInstance = instance => {
         return isNumber(value) ? value : getAttachmentLimit(software);
       });
 
-      // Merge defaults & cleanup
+      // Merge defaults
       instance.mergeDeepWith(mergeDefined, InstanceRecord());
-      instance.deleteAll(['max_toot_chars', 'poll_limits']);
     }),
   );
 };

app/soapbox/normalizers/status.ts

@@ -1,12 +1,14 @@
+import escapeTextContentForBrowser from 'escape-html';
 import {
   Map as ImmutableMap,
   List as ImmutableList,
   Record as ImmutableRecord,
 } from 'immutable';
 
+import emojify from 'soapbox/features/emoji/emoji';
+import { normalizeAccount } from 'soapbox/normalizers/account';
 import { IStatus } from 'soapbox/types';
-import { accountToMention } from 'soapbox/utils/accounts';
+import { mergeDefined, makeEmojiMap } from 'soapbox/utils/normalizers';
-import { mergeDefined } from 'soapbox/utils/normalizers';
 
 const StatusRecord = ImmutableRecord({
   account: ImmutableMap(),
@@ -47,9 +49,25 @@ const StatusRecord = ImmutableRecord({
   spoilerHtml: '',
 });
 
-const PollOptionRecord = ImmutableRecord({
+// https://docs.joinmastodon.org/entities/attachment/
-  title: '',
+const AttachmentRecord = ImmutableRecord({
-  votes_count: 0,
+  blurhash: undefined,
+  description: '',
+  id: '',
+  meta: ImmutableMap(),
+  pleroma: ImmutableMap(),
+  preview_url: '',
+  remote_url: null,
+  type: 'unknown',
+  url: '',
+});
+
+// https://docs.joinmastodon.org/entities/mention/
+const MentionRecord = ImmutableRecord({
+  id: '',
+  acct: '',
+  username: '',
+  url: '',
 });
 
 // https://docs.joinmastodon.org/entities/poll/
@@ -66,6 +84,24 @@ const PollRecord = ImmutableRecord({
   voted: false,
 });
 
+// Sub-entity of Poll
+const PollOptionRecord = ImmutableRecord({
+  title: '',
+  votes_count: 0,
+
+  // Internal fields
+  title_emojified: '',
+});
+
+// https://docs.joinmastodon.org/entities/emoji/
+const EmojiRecord = ImmutableRecord({
+  category: '',
+  shortcode: '',
+  static_url: '',
+  url: '',
+  visible_in_picker: true,
+});
+
 // Ensure attachments have required fields
 // https://docs.joinmastodon.org/entities/attachment/
 const normalizeAttachment = (attachment: ImmutableMap<string, any>) => {
@@ -78,10 +114,9 @@ const normalizeAttachment = (attachment: ImmutableMap<string, any>) => {
   const base = ImmutableMap({
     url,
     preview_url: url,
-    remote_url: url,
   });
 
-  return attachment.mergeWith(mergeDefined, base);
+  return AttachmentRecord(attachment.mergeWith(mergeDefined, base));
 };
 
 const normalizeAttachments = (status: ImmutableMap<string, any>) => {
@@ -92,13 +127,7 @@ const normalizeAttachments = (status: ImmutableMap<string, any>) => {
 
 // Normalize mentions
 const normalizeMention = (mention: ImmutableMap<string, any>) => {
-  const base = ImmutableMap({
+  return MentionRecord(normalizeAccount(mention));
-    acct: '',
-    username: (mention.get('acct') || '').split('@')[0],
-    url: '',
-  });
-
-  return mention.mergeWith(mergeDefined, base);
 };
 
 const normalizeMentions = (status: ImmutableMap<string, any>) => {
@@ -107,10 +136,28 @@ const normalizeMentions = (status: ImmutableMap<string, any>) => {
   });
 };
 
+// Normalize emojis
+const normalizeEmojis = (entity: ImmutableMap<string, any>) => {
+  return entity.update('emojis', ImmutableList(), emojis => {
+    return emojis.map(EmojiRecord);
+  });
+};
+
+const normalizePollOption = (option: ImmutableMap<string, any>, emojis: ImmutableList<ImmutableMap<string, string>> = ImmutableList()) => {
+  const emojiMap = makeEmojiMap(emojis);
+  const titleEmojified = emojify(escapeTextContentForBrowser(option.get('title')), emojiMap);
+
+  return PollOptionRecord(
+    option.set('title_emojified', titleEmojified),
+  );
+};
+
 // Normalize poll options
 const normalizePollOptions = (poll: ImmutableMap<string, any>) => {
+  const emojis = poll.get('emojis');
+
   return poll.update('options', (options: ImmutableList<ImmutableMap<string, any>>) => {
-    return options.map(PollOptionRecord);
+    return options.map(option => normalizePollOption(option, emojis));
   });
 };
 
@@ -132,6 +179,7 @@ const normalizePollVoted = (poll: ImmutableMap<string, any>) => {
 const normalizePoll = (poll: ImmutableMap<string, any>) => {
   return PollRecord(
     poll.withMutations((poll: ImmutableMap<string, any>) => {
+      normalizeEmojis(poll);
       normalizePollOptions(poll);
       normalizePollOwnVotes(poll);
       normalizePollVoted(poll);
@@ -172,8 +220,8 @@ const addSelfMention = (status: ImmutableMap<string, any>) => {
   const isSelfReply = accountId === status.get('in_reply_to_account_id');
   const hasSelfMention = accountId === status.getIn(['mentions', 0, 'id']);
 
-  if (isSelfReply && !hasSelfMention) {
+  if (isSelfReply && !hasSelfMention && accountId) {
-    const mention = accountToMention(status.get('account'));
+    const mention = normalizeMention(status.get('account'));
     return status.update('mentions', ImmutableList(), mentions => (
       ImmutableList([mention]).concat(mentions)
     ));
@@ -195,6 +243,7 @@ export const normalizeStatus = (status: ImmutableMap<string, any>): IStatus => {
     status.withMutations(status => {
       normalizeAttachments(status);
       normalizeMentions(status);
+      normalizeEmojis(status);
       normalizeStatusPoll(status);
       fixMentionsOrder(status);
       addSelfMention(status);

app/soapbox/reducers/__tests__/statuses-test.js

@@ -1,4 +1,8 @@
-import { Map as ImmutableMap, Record, fromJS } from 'immutable';
+import {
+  Map as ImmutableMap,
+  Record as ImmutableRecord,
+  fromJS,
+} from 'immutable';
 
 import { STATUS_IMPORT } from 'soapbox/actions/importer';
 import {
@@ -19,7 +23,7 @@ describe('statuses reducer', () => {
       const action = { type: STATUS_IMPORT, status };
       const result = reducer(undefined, action).get('AFmFMSpITT9xcOJKcK');
 
-      expect(Record.isRecord(result)).toBe(true);
+      expect(ImmutableRecord.isRecord(result)).toBe(true);
     });
 
     it('fixes the order of mentions', () => {
@@ -52,42 +56,45 @@ describe('statuses reducer', () => {
 
       const state = reducer(undefined, { type: STATUS_IMPORT, status });
 
-      const expected = fromJS([{
+      const expected = [{
         id: '017eeb0e-e5df-30a4-77a7-a929145cb836',
         type: 'image',
         url: 'https://mitra.social/media/8e04e6091bbbac79641b5812508683ce72c38693661c18d16040553f2371e18d.png',
         preview_url: 'https://mitra.social/media/8e04e6091bbbac79641b5812508683ce72c38693661c18d16040553f2371e18d.png',
-        remote_url: 'https://mitra.social/media/8e04e6091bbbac79641b5812508683ce72c38693661c18d16040553f2371e18d.png',
+        remote_url: null,
       }, {
         id: '017eeb0e-e5e4-2a48-2889-afdebf368a54',
         type: 'unknown',
         url: 'https://mitra.social/media/8f72dc2e98572eb4ba7c3a902bca5f69c448fc4391837e5f8f0d4556280440ac',
         preview_url: 'https://mitra.social/media/8f72dc2e98572eb4ba7c3a902bca5f69c448fc4391837e5f8f0d4556280440ac',
-        remote_url: 'https://mitra.social/media/8f72dc2e98572eb4ba7c3a902bca5f69c448fc4391837e5f8f0d4556280440ac',
+        remote_url: null,
       }, {
         id: '017eeb0e-e5e5-79fd-6054-8b6869b1db49',
         type: 'unknown',
         url: 'https://mitra.social/media/55a81a090247cc4fc127e5716bcf7964f6e0df9b584f85f4696c0b994747a4d0.oga',
         preview_url: 'https://mitra.social/media/55a81a090247cc4fc127e5716bcf7964f6e0df9b584f85f4696c0b994747a4d0.oga',
-        remote_url: 'https://mitra.social/media/55a81a090247cc4fc127e5716bcf7964f6e0df9b584f85f4696c0b994747a4d0.oga',
+        remote_url: null,
       }, {
         id: '017eeb0e-e5e6-c416-a444-21e560c47839',
         type: 'unknown',
         url: 'https://mitra.social/media/0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0',
         preview_url: 'https://mitra.social/media/0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0',
-        remote_url: 'https://mitra.social/media/0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0',
+        remote_url: null,
-      }]);
+      }];
 
-      expect(state.getIn(['017eeb0e-e5e7-98fe-6b2b-ad02349251fb', 'media_attachments'])).toEqual(expected);
+      expect(state.getIn(['017eeb0e-e5e7-98fe-6b2b-ad02349251fb', 'media_attachments']).toJS()).toMatchObject(expected);
     });
 
-    it('leaves Pleroma attachments alone', () => {
+    it('fixes Pleroma attachments', () => {
       const status = require('soapbox/__fixtures__/pleroma-status-with-attachments.json');
       const action = { type: STATUS_IMPORT, status };
       const state = reducer(undefined, action);
-      const expected = fromJS(status.media_attachments);
+      const result = state.get('AGNkA21auFR5lnEAHw').media_attachments;
 
-      expect(state.getIn(['AGNkA21auFR5lnEAHw', 'media_attachments'])).toEqual(expected);
+      expect(result.size).toBe(4);
+      expect(result.get(0).text_url).toBe(undefined);
+      expect(result.get(1).meta).toEqual(ImmutableMap());
+      expect(result.getIn([1, 'pleroma', 'mime_type'])).toBe('application/x-nes-rom');
     });
 
     it('hides CWs', () => {

app/soapbox/reducers/statuses.js

@@ -5,6 +5,7 @@ import emojify from 'soapbox/features/emoji/emoji';
 import { normalizeStatus } from 'soapbox/normalizers/status';
 import { simulateEmojiReact, simulateUnEmojiReact } from 'soapbox/utils/emoji_reacts';
 import { stripCompatibilityFeatures } from 'soapbox/utils/html';
+import { makeEmojiMap } from 'soapbox/utils/normalizers';
 
 import {
   EMOJI_REACT_REQUEST,
@@ -32,11 +33,6 @@ import { TIMELINE_DELETE } from '../actions/timelines';
 
 const domParser = new DOMParser();
 
-const makeEmojiMap = record => record.get('emojis').reduce((obj, emoji) => {
-  obj[`:${emoji.get('shortcode')}:`] = emoji.toJS();
-  return obj;
-}, {});
-
 const minifyStatus = status => {
   return status.mergeWith((o, n) => n || o, {
     account: status.getIn(['account', 'id']),
@@ -59,7 +55,7 @@ export const calculateStatus = (status, oldStatus, expandSpoilers = false) => {
   } else {
     const spoilerText   = status.get('spoiler_text') || '';
     const searchContent = (ImmutableList([spoilerText, status.get('content')]).concat(status.getIn(['poll', 'options'], ImmutableList()).map(option => option.get('title')))).join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
-    const emojiMap      = makeEmojiMap(status);
+    const emojiMap      = makeEmojiMap(status.get('emojis'));
 
     return status.merge({
       search_index: domParser.parseFromString(searchContent, 'text/html').documentElement.textContent,

app/soapbox/utils/__tests__/accounts-test.js

@@ -6,7 +6,6 @@ import {
   isStaff,
   isAdmin,
   isModerator,
-  accountToMention,
 } from '../accounts';
 
 describe('getDomain', () => {
@@ -116,19 +115,3 @@ describe('isModerator', () => {
     });
   });
 });
-
-describe('accountToMention', () => {
-  it('converts the account to a mention', () => {
-    const account = fromJS(require('soapbox/__fixtures__/pleroma-account.json'));
-
-    const expected = fromJS({
-      id: '9v5bmRalQvjOy0ECcC',
-      username: 'alex',
-      acct: 'alex',
-      url: 'https://gleasonator.com/users/alex',
-    });
-
-    const result = accountToMention(account);
-    expect(result).toEqual(expected);
-  });
-});

app/soapbox/utils/accounts.ts

@@ -62,12 +62,3 @@ export const isLocal = (account: ImmutableMap<string, any>): boolean => {
 };
 
 export const isRemote = (account: ImmutableMap<string, any>): boolean => !isLocal(account);
-
-export const accountToMention = (account: ImmutableMap<string, any>): ImmutableMap<string, any> => {
-  return ImmutableMap({
-    id: account.get('id'),
-    username: account.get('username'),
-    acct: account.get('acct'),
-    url: account.get('url'),
-  });
-};

app/soapbox/utils/normalizers.js

@@ -1,2 +1,7 @@
 // Use new value only if old value is undefined
 export const mergeDefined = (oldVal, newVal) => oldVal === undefined ? newVal : oldVal;
+
+export const makeEmojiMap = emojis => emojis.reduce((obj, emoji) => {
+  obj[`:${emoji.shortcode}:`] = emoji;
+  return obj;
+}, {});

package.json

@@ -58,6 +58,7 @@
     "@sentry/react": "^6.12.0",
     "@sentry/tracing": "^6.12.0",
     "@tabler/icons": "^1.53.0",
+    "@types/escape-html": "^1.0.1",
     "array-includes": "^3.0.3",
     "autoprefixer": "^10.0.0",
     "axios": "^0.21.4",

yarn.lock

@@ -1747,6 +1747,11 @@
   dependencies:
     "@babel/types" "^7.3.0"
 
+"@types/escape-html@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@types/escape-html/-/escape-html-1.0.1.tgz#b19b4646915f0ae2c306bf984dc0a59c5cfc97ba"
+  integrity sha512-4mI1FuUUZiuT95fSVqvZxp/ssQK9zsa86S43h9x3zPOSU9BBJ+BfDkXwuaU7BfsD+e7U0/cUUfJFk3iW2M4okA==
+
 "@types/eslint-scope@^3.7.0":
   version "3.7.1"
   resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.1.tgz#8dc390a7b4f9dd9f1284629efce982e41612116e"