Merge branch 'more-entity-normalization' into 'develop'

More entity normalization

See merge request soapbox-pub/soapbox-fe!1091

app/soapbox/__fixtures__/pleroma-status-with-poll-with-emojis.json

@@ -0,0 +1,236 @@
+{
+  "account": {
+    "acct": "alex",
+    "avatar": "https://media.gleasonator.com/6d64aecb17348b23aaff78db4687b9476cb0da1c07cc6a819c2e6ec7144c18b1.png",
+    "avatar_static": "https://media.gleasonator.com/6d64aecb17348b23aaff78db4687b9476cb0da1c07cc6a819c2e6ec7144c18b1.png",
+    "bot": false,
+    "created_at": "2020-01-08T01:25:43.000Z",
+    "display_name": "Alex Gleason",
+    "emojis": [],
+    "fields": [
+      {
+        "name": "Website",
+        "value": "<a href=\"https://alexgleason.me\" rel=\"ugc\">https://alexgleason.me</a>"
+      },
+      {
+        "name": "Soapbox",
+        "value": "<a href=\"https://soapbox.pub\" rel=\"ugc\">https://soapbox.pub</a>"
+      },
+      {
+        "name": "Email",
+        "value": "alex@alexgleason.me"
+      },
+      {
+        "name": "Gender identity",
+        "value": "Soyboy"
+      },
+      {
+        "name": "Donate (PayPal)",
+        "value": "<a href=\"https://paypal.me/gleasonator\" rel=\"ugc\">https://paypal.me/gleasonator</a>"
+      },
+      {
+        "name": "$BTC",
+        "value": "bc1q9cx35adpm73aq2fw40ye6ts8hfxqzjr5unwg0n"
+      },
+      {
+        "name": "$ETH",
+        "value": "0xAc9aB5Fc04Dc1cB1789Af75b523Bd23C70B2D717"
+      },
+      {
+        "name": "$DOGE",
+        "value": "D5zVZs6jrRakaPVGiErkQiHt9sayzm6V5D"
+      },
+      {
+        "name": "$XMR",
+        "value": "45JDCLrjJ4bgVUSbbs2yjy9m5Mf4VLPW8fG7jw9sq5u69rXZZopQogZNeyYkMBnXpkaip4p4QwaaJNhdTotPa9g44DBCzdK"
+      }
+    ],
+    "followers_count": 2467,
+    "following_count": 1581,
+    "fqn": "alex@gleasonator.com",
+    "header": "https://media.gleasonator.com/accounts/headers/000/000/001/original/9d0e4dbf1c9dbc8f.png",
+    "header_static": "https://media.gleasonator.com/accounts/headers/000/000/001/original/9d0e4dbf1c9dbc8f.png",
+    "id": "9v5bmRalQvjOy0ECcC",
+    "last_status_at": "2022-03-11T01:33:19",
+    "locked": false,
+    "note": "I create Fediverse software that empowers people online.<br/><br/>I&#39;m vegan btw<br/><br/>Note: If you have a question for me, please tag me publicly. This gives the opportunity for others to chime in, and bystanders to learn.",
+    "pleroma": {
+      "accepts_chat_messages": true,
+      "also_known_as": [
+        "https://mitra.social/users/alex"
+      ],
+      "ap_id": "https://gleasonator.com/users/alex",
+      "background_image": null,
+      "birthday": "1993-07-03",
+      "favicon": "https://gleasonator.com/favicon.png",
+      "hide_favorites": true,
+      "hide_followers": false,
+      "hide_followers_count": false,
+      "hide_follows": false,
+      "hide_follows_count": false,
+      "is_admin": true,
+      "is_confirmed": true,
+      "is_moderator": false,
+      "is_suggested": true,
+      "relationship": {},
+      "skip_thread_containment": false,
+      "tags": []
+    },
+    "source": {
+      "fields": [
+        {
+          "name": "Website",
+          "value": "https://alexgleason.me"
+        },
+        {
+          "name": "Soapbox",
+          "value": "https://soapbox.pub"
+        },
+        {
+          "name": "Email",
+          "value": "alex@alexgleason.me"
+        },
+        {
+          "name": "Gender identity",
+          "value": "Soyboy"
+        },
+        {
+          "name": "Donate (PayPal)",
+          "value": "https://paypal.me/gleasonator"
+        },
+        {
+          "name": "$BTC",
+          "value": "bc1q9cx35adpm73aq2fw40ye6ts8hfxqzjr5unwg0n"
+        },
+        {
+          "name": "$ETH",
+          "value": "0xAc9aB5Fc04Dc1cB1789Af75b523Bd23C70B2D717"
+        },
+        {
+          "name": "$DOGE",
+          "value": "D5zVZs6jrRakaPVGiErkQiHt9sayzm6V5D"
+        },
+        {
+          "name": "$XMR",
+          "value": "45JDCLrjJ4bgVUSbbs2yjy9m5Mf4VLPW8fG7jw9sq5u69rXZZopQogZNeyYkMBnXpkaip4p4QwaaJNhdTotPa9g44DBCzdK"
+        }
+      ],
+      "note": "I create Fediverse software that empowers people online.\r\n\r\nI'm vegan btw\r\n\r\nNote: If you have a question for me, please tag me publicly. This gives the opportunity for others to chime in, and bystanders to learn.",
+      "pleroma": {
+        "actor_type": "Person",
+        "discoverable": false
+      },
+      "sensitive": false
+    },
+    "statuses_count": 23651,
+    "url": "https://gleasonator.com/users/alex",
+    "username": "alex"
+  },
+  "application": {
+    "name": "Soapbox FE",
+    "website": "https://soapbox.pub/"
+  },
+  "bookmarked": false,
+  "card": null,
+  "content": "<p>Test poll</p>",
+  "created_at": "2022-03-11T01:33:18.000Z",
+  "emojis": [
+    {
+      "shortcode": "gleason_excited",
+      "static_url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+      "url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+      "visible_in_picker": false
+    },
+    {
+      "shortcode": "soapbox",
+      "static_url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+      "url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+      "visible_in_picker": false
+    }
+  ],
+  "favourited": false,
+  "favourites_count": 1,
+  "id": "AHHue68kB59xtUv7MO",
+  "in_reply_to_account_id": null,
+  "in_reply_to_id": null,
+  "language": null,
+  "media_attachments": [],
+  "mentions": [],
+  "muted": false,
+  "pinned": false,
+  "pleroma": {
+    "content": {
+      "text/plain": "Test poll"
+    },
+    "conversation_id": "AHHue65YMwbjjbQZO4",
+    "direct_conversation_id": null,
+    "emoji_reactions": [],
+    "expires_at": null,
+    "in_reply_to_account_acct": null,
+    "local": true,
+    "parent_visible": false,
+    "pinned_at": null,
+    "quote": null,
+    "quote_url": null,
+    "quote_visible": false,
+    "spoiler_text": {
+      "text/plain": ""
+    },
+    "thread_muted": false
+  },
+  "poll": {
+    "emojis": [
+      {
+        "shortcode": "gleason_excited",
+        "static_url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+        "url": "https://gleasonator.com/emoji/gleason_emojis/gleason_excited.png",
+        "visible_in_picker": false
+      },
+      {
+        "shortcode": "soapbox",
+        "static_url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+        "url": "https://gleasonator.com/emoji/Gleasonator/soapbox.png",
+        "visible_in_picker": false
+      }
+    ],
+    "expired": false,
+    "expires_at": "2022-03-12T01:33:18.000Z",
+    "id": "AHHue67gF2JDqCQGhc",
+    "multiple": false,
+    "options": [
+      {
+        "title": "Regular emoji 😍 ",
+        "votes_count": 0
+      },
+      {
+        "title": "Custom emoji :gleason_excited: ",
+        "votes_count": 1
+      },
+      {
+        "title": "No emoji",
+        "votes_count": 0
+      },
+      {
+        "title": "🤔 😮 😠 ",
+        "votes_count": 1
+      },
+      {
+        "title": ":soapbox:",
+        "votes_count": 1
+      }
+    ],
+    "voters_count": 3,
+    "votes_count": 3
+  },
+  "reblog": null,
+  "reblogged": false,
+  "reblogs_count": 1,
+  "replies_count": 1,
+  "sensitive": false,
+  "spoiler_text": "",
+  "tags": [],
+  "text": null,
+  "uri": "https://gleasonator.com/objects/46d2ab26-3497-442b-999f-612fe717b0a3",
+  "url": "https://gleasonator.com/notice/AHHue68kB59xtUv7MO",
+  "visibility": "public"
+}

app/soapbox/normalizers/account.ts

@@ -97,6 +97,13 @@ const normalizeLocation = (account: ImmutableMap<string, any>) => {
   });
 };
 
+// Set username from acct, if applicable
+const fixUsername = (account: ImmutableMap<string, any>) => {
+  return account.update('username', username => (
+    username || (account.get('acct') || '').split('@')[0]
+  ));
+};
+
 export const normalizeAccount = (account: ImmutableMap<string, any>): IAccount => {
   return AccountRecord(
     account.withMutations(account => {
@@ -104,6 +111,7 @@ export const normalizeAccount = (account: ImmutableMap<string, any>): IAccount =
       normalizeVerified(account);
       normalizeBirthday(account);
       normalizeLocation(account);
+      fixUsername(account);
     }),
   );
 };

app/soapbox/normalizers/status.ts

@@ -1,12 +1,14 @@
+import escapeTextContentForBrowser from 'escape-html';
 import {
   Map as ImmutableMap,
   List as ImmutableList,
   Record as ImmutableRecord,
 } from 'immutable';
 
+import emojify from 'soapbox/features/emoji/emoji';
+import { normalizeAccount } from 'soapbox/normalizers/account';
 import { IStatus } from 'soapbox/types';
-import { accountToMention } from 'soapbox/utils/accounts';
+import { mergeDefined, makeEmojiMap } from 'soapbox/utils/normalizers';
-import { mergeDefined } from 'soapbox/utils/normalizers';
 
 const StatusRecord = ImmutableRecord({
   account: ImmutableMap(),
@@ -47,9 +49,25 @@ const StatusRecord = ImmutableRecord({
   spoilerHtml: '',
 });
 
-const PollOptionRecord = ImmutableRecord({
+// https://docs.joinmastodon.org/entities/attachment/
-  title: '',
+const AttachmentRecord = ImmutableRecord({
-  votes_count: 0,
+  blurhash: undefined,
+  description: '',
+  id: '',
+  meta: ImmutableMap(),
+  pleroma: ImmutableMap(),
+  preview_url: '',
+  remote_url: null,
+  type: 'unknown',
+  url: '',
+});
+
+// https://docs.joinmastodon.org/entities/mention/
+const MentionRecord = ImmutableRecord({
+  id: '',
+  acct: '',
+  username: '',
+  url: '',
 });
 
 // https://docs.joinmastodon.org/entities/poll/
@@ -66,6 +84,24 @@ const PollRecord = ImmutableRecord({
   voted: false,
 });
 
+// Sub-entity of Poll
+const PollOptionRecord = ImmutableRecord({
+  title: '',
+  votes_count: 0,
+
+  // Internal fields
+  title_emojified: '',
+});
+
+// https://docs.joinmastodon.org/entities/emoji/
+const EmojiRecord = ImmutableRecord({
+  category: '',
+  shortcode: '',
+  static_url: '',
+  url: '',
+  visible_in_picker: true,
+});
+
 // Ensure attachments have required fields
 // https://docs.joinmastodon.org/entities/attachment/
 const normalizeAttachment = (attachment: ImmutableMap<string, any>) => {
@@ -78,10 +114,9 @@ const normalizeAttachment = (attachment: ImmutableMap<string, any>) => {
   const base = ImmutableMap({
     url,
     preview_url: url,
-    remote_url: url,
   });
 
-  return attachment.mergeWith(mergeDefined, base);
+  return AttachmentRecord(attachment.mergeWith(mergeDefined, base));
 };
 
 const normalizeAttachments = (status: ImmutableMap<string, any>) => {
@@ -92,13 +127,7 @@ const normalizeAttachments = (status: ImmutableMap<string, any>) => {
 
 // Normalize mentions
 const normalizeMention = (mention: ImmutableMap<string, any>) => {
-  const base = ImmutableMap({
+  return MentionRecord(normalizeAccount(mention));
-    acct: '',
-    username: (mention.get('acct') || '').split('@')[0],
-    url: '',
-  });
-
-  return mention.mergeWith(mergeDefined, base);
 };
 
 const normalizeMentions = (status: ImmutableMap<string, any>) => {
@@ -107,10 +136,28 @@ const normalizeMentions = (status: ImmutableMap<string, any>) => {
   });
 };
 
+// Normalize emojis
+const normalizeEmojis = (entity: ImmutableMap<string, any>) => {
+  return entity.update('emojis', ImmutableList(), emojis => {
+    return emojis.map(EmojiRecord);
+  });
+};
+
+const normalizePollOption = (option: ImmutableMap<string, any>, emojis: ImmutableList<ImmutableMap<string, string>> = ImmutableList()) => {
+  const emojiMap = makeEmojiMap(emojis);
+  const titleEmojified = emojify(escapeTextContentForBrowser(option.get('title')), emojiMap);
+
+  return PollOptionRecord(
+    option.set('title_emojified', titleEmojified),
+  );
+};
+
 // Normalize poll options
 const normalizePollOptions = (poll: ImmutableMap<string, any>) => {
+  const emojis = poll.get('emojis');
+
   return poll.update('options', (options: ImmutableList<ImmutableMap<string, any>>) => {
-    return options.map(PollOptionRecord);
+    return options.map(option => normalizePollOption(option, emojis));
   });
 };
 
@@ -132,6 +179,7 @@ const normalizePollVoted = (poll: ImmutableMap<string, any>) => {
 const normalizePoll = (poll: ImmutableMap<string, any>) => {
   return PollRecord(
     poll.withMutations((poll: ImmutableMap<string, any>) => {
+      normalizeEmojis(poll);
       normalizePollOptions(poll);
       normalizePollOwnVotes(poll);
       normalizePollVoted(poll);
@@ -172,8 +220,8 @@ const addSelfMention = (status: ImmutableMap<string, any>) => {
   const isSelfReply = accountId === status.get('in_reply_to_account_id');
   const hasSelfMention = accountId === status.getIn(['mentions', 0, 'id']);
 
-  if (isSelfReply && !hasSelfMention) {
+  if (isSelfReply && !hasSelfMention && accountId) {
-    const mention = accountToMention(status.get('account'));
+    const mention = normalizeMention(status.get('account'));
     return status.update('mentions', ImmutableList(), mentions => (
       ImmutableList([mention]).concat(mentions)
     ));
@@ -195,6 +243,7 @@ export const normalizeStatus = (status: ImmutableMap<string, any>): IStatus => {
     status.withMutations(status => {
       normalizeAttachments(status);
       normalizeMentions(status);
+      normalizeEmojis(status);
       normalizeStatusPoll(status);
       fixMentionsOrder(status);
       addSelfMention(status);

app/soapbox/utils/accounts.ts

@@ -62,12 +62,3 @@ export const isLocal = (account: ImmutableMap<string, any>): boolean => {
 };
 
 export const isRemote = (account: ImmutableMap<string, any>): boolean => !isLocal(account);
-
-export const accountToMention = (account: ImmutableMap<string, any>): ImmutableMap<string, any> => {
-  return ImmutableMap({
-    id: account.get('id'),
-    username: account.get('username'),
-    acct: account.get('acct'),
-    url: account.get('url'),
-  });
-};

package.json

@@ -58,6 +58,7 @@
     "@sentry/react": "^6.12.0",
     "@sentry/tracing": "^6.12.0",
     "@tabler/icons": "^1.53.0",
+    "@types/escape-html": "^1.0.1",
     "array-includes": "^3.0.3",
     "autoprefixer": "^10.0.0",
     "axios": "^0.21.4",

yarn.lock

@@ -1747,6 +1747,11 @@
   dependencies:
     "@babel/types" "^7.3.0"
 
+"@types/escape-html@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@types/escape-html/-/escape-html-1.0.1.tgz#b19b4646915f0ae2c306bf984dc0a59c5cfc97ba"
+  integrity sha512-4mI1FuUUZiuT95fSVqvZxp/ssQK9zsa86S43h9x3zPOSU9BBJ+BfDkXwuaU7BfsD+e7U0/cUUfJFk3iW2M4okA==
+
 "@types/eslint-scope@^3.7.0":
   version "3.7.1"
   resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.1.tgz#8dc390a7b4f9dd9f1284629efce982e41612116e"