bigbuffet/pleroma
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Get basic embed working

Browse source
This commit is contained in:
Alex Gleason 2021-05-04 02:23:22 -05:00
parent b221d77a6d
commit 765bdfca7d
No known key found for this signature in database
GPG key ID: 7211D1F99744FBB7
8 changed files with 131 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
config/config.exs
View file

@ -409,8 +409,8 @@
ignore_hosts: [],
ignore_tld: ["local", "localdomain", "lan"],
parsers: [
Pleroma.Web.RichMedia.Parsers.TwitterCard,
Pleroma.Web.RichMedia.Parsers.OEmbed
Pleroma.Web.RichMedia.Parsers.OEmbed,
Pleroma.Web.RichMedia.Parsers.TwitterCard
],
failure_backoff: 60_000,
ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]

32
lib/pleroma/web/mastodon_api/views/status_view.ex
View file

@ -367,36 +367,8 @@ def render("show.json", _) do
nil
end
def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
page_url_data = URI.parse(page_url)
page_url_data =
if is_binary(rich_media["url"]) do
URI.merge(page_url_data, URI.parse(rich_media["url"]))
else
page_url_data
end
page_url = page_url_data |> to_string
image_url =
if is_binary(rich_media["image"]) do
URI.merge(page_url_data, URI.parse(rich_media["image"]))
|> to_string
end
%{
type: "link",
provider_name: page_url_data.host,
provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
url: page_url,
image: image_url |> MediaProxy.url(),
title: rich_media["title"] || "",
description: rich_media["description"] || "",
pleroma: %{
opengraph: rich_media
}
}
def render("card.json", %{rich_media: rich_media, page_url: _page_url}) do
rich_media
end
def render("card.json", _), do: nil

3
lib/pleroma/web/plugs/http_security_plug.ex
View file

@ -79,7 +79,8 @@ def headers do
"frame-ancestors 'none'",
"style-src 'self' 'unsafe-inline'",
"font-src 'self'",
"manifest-src 'self'"
"manifest-src 'self'",
"frame-src 'self' https:"
]
@csp_start [Enum.join(static_csp_rules, ";") <> ";"]

22
lib/pleroma/web/rich_media/helpers.ex
View file

@ -8,6 +8,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.HTML
alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser
alias Pleroma.Web.RichMedia.Parser.Card
@options [
pool: :media,
@ -15,6 +16,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do
recv_timeout: 2_000
]
@headers [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
@spec validate_page_url(URI.t() | binary()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Config.get([Pleroma.Formatter, :validate_tld])
@ -55,12 +58,23 @@ defp get_tld(host) do
|> hd
end
defp strip_card(%Card{} = card) do
card
|> Map.from_struct()
|> Map.new(fn {k, v} -> {Atom.to_string(k), v} end)
end
defp strip_card(%{} = card) do
Map.new(card, fn {k, v} -> {Atom.to_string(k), v} end)
end
def fetch_data_for_object(object) do
with true <- Config.get([:rich_media, :enabled]),
{:ok, page_url} <-
HTML.extract_first_external_url_from_object(object),
:ok <- validate_page_url(page_url),
{:ok, rich_media} <- Parser.parse(page_url) do
{:ok, rich_media} <- Parser.parse(page_url),
rich_media <- strip_card(rich_media) do
%{page_url: page_url, rich_media: rich_media}
else
_ -> %{}
@ -78,8 +92,12 @@ def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) d
def fetch_data_for_activity(_), do: %{}
def oembed_get(url) do
Pleroma.HTTP.get(url, @headers, @options)
end
def rich_media_get(url) do
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
headers = @headers
head_check =
case Pleroma.HTTP.head(url, headers, @options) do

32
lib/pleroma/web/rich_media/parser.ex
View file

@ -4,6 +4,7 @@
defmodule Pleroma.Web.RichMedia.Parser do
require Logger
alias Pleroma.Web.RichMedia.Parser.Card
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
@ -131,13 +132,34 @@ defp get_ttl_from_image(data, url) do
end
def parse_url(url) do
case maybe_fetch_oembed(url) do
{:ok, %Card{} = card} -> {:ok, card}
_ -> fetch_document(url)
end
end
defp maybe_fetch_oembed(url) do
with {:ok, oembed_url} <- OEmbedProviders.oembed_url(url),
{:ok, %Tesla.Env{body: json}} <-
Pleroma.Web.RichMedia.Helpers.oembed_get(oembed_url),
{:ok, data} <- Jason.decode(json),
%Card{} = card <- Card.from_oembed(data, url) do
{:ok, card}
else
{:error, error} -> {:error, error}
error -> {:error, error}
end
end
defp fetch_document(url) do
with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
{:ok, html} <- Floki.parse_document(html) do
html
|> maybe_parse()
|> Map.put("url", url)
|> clean_parsed_data()
|> check_parsed_data()
|> Card.from_meta_tags(url)
|> check_card()
end
end
@ -150,13 +172,13 @@ defp maybe_parse(html) do
end)
end
defp check_parsed_data(%{"title" => title} = data)
defp check_card(%Card{title: title} = card)
when is_binary(title) and title != "" do
{:ok, data}
{:ok, card}
end
defp check_parsed_data(data) do
{:error, {:invalid_metadata, data}}
defp check_card(card) do
{:error, {:invalid_metadata, card}}
end
defp clean_parsed_data(data) do

75
lib/pleroma/web/rich_media/parser/card.ex Normal file
View file

@ -0,0 +1,75 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser.Card do
@types ["link", "photo", "video", "rich"]
# https://docs.joinmastodon.org/entities/card/
defstruct url: nil,
title: nil,
description: "",
type: "link",
author_name: "",
author_url: "",
provider_name: "",
provider_url: "",
html: "",
width: 0,
height: 0,
image: nil,
embed_url: "",
blurhash: nil
def from_oembed(%{"type" => type, "title" => title} = oembed, url) when type in @types do
%__MODULE__{
url: url,
title: title,
description: "",
type: type,
author_name: oembed["author_name"],
author_url: oembed["author_url"],
provider_name: oembed["provider_name"],
provider_url: oembed["provider_url"],
html: oembed["html"],
width: oembed["width"],
height: oembed["height"],
image: oembed["thumbnail_url"] |> proxy(),
embed_url: oembed["url"] |> proxy()
}
end
def from_oembed(_oembed, _url), do: nil
def from_meta_tags(rich_media, page_url) do
page_url_data = URI.parse(page_url)
page_url_data =
if is_binary(rich_media["url"]) do
URI.merge(page_url_data, URI.parse(rich_media["url"]))
else
page_url_data
end
page_url = page_url_data |> to_string
image_url =
if is_binary(rich_media["image"]) do
URI.merge(page_url_data, URI.parse(rich_media["image"]))
|> to_string
end
%__MODULE__{
type: "link",
provider_name: page_url_data.host,
provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
url: page_url,
image: image_url |> proxy(),
title: rich_media["title"] || "",
description: rich_media["description"] || ""
}
end
defp proxy(url) when is_binary(url), do: Pleroma.Web.MediaProxy.url(url)
defp proxy(_), do: nil
end

1
mix.exs
View file

@ -198,6 +198,7 @@ defp deps do
{:open_api_spex,
git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"},
{:oembed_providers, "~> 0.1.0"},
## dev & test
{:ex_doc, "~> 0.22", only: :dev, runtime: false},

2
mix.lock
View file

@ -52,6 +52,7 @@
"gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"},
"gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
"gettext": {:hex, :gettext, "0.18.0", "406d6b9e0e3278162c2ae1de0a60270452c553536772167e2d701f028116f870", [:mix], [], "hexpm", "c3f850be6367ebe1a08616c2158affe4a23231c70391050bf359d5f92f66a571"},
"glob": {:hex, :glob, "1.0.0", "b4d54d66e7797ce037cdd18f2587fc9932187355340e222cafe125cd333d7a0a", [:rebar3], [], "hexpm", "ca25de25ac5a762ba6c979718ae6afef8402cfc9155b87479d215fbe676801e1"},
"gun": {:git, "https://github.com/ninenines/gun.git", "921c47146b2d9567eac7e9a4d2ccc60fffd4f327", [ref: "921c47146b2d9567eac7e9a4d2ccc60fffd4f327"]},
"hackney": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/hackney.git", "7d7119f0651515d6d7669c78393fd90950a3ec6e", [ref: "7d7119f0651515d6d7669c78393fd90950a3ec6e"]},
"html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"},
@ -82,6 +83,7 @@
"nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"},
"nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
"oban": {:hex, :oban, "2.3.4", "ec7509b9af2524d55f529cb7aee93d36131ae0bf0f37706f65d2fe707f4d9fd8", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c70ca0434758fd1805422ea4446af5e910ddc697c0c861549c8f0eb0cfbd2fdf"},
"oembed_providers": {:hex, :oembed_providers, "0.1.0", "9b336ee5f3ca20ee4ed005383c74b154d30d0abeb98e95828855c0e2841ae46b", [:mix], [{:glob, "~> 1.0", [hex: :glob, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "ac1dda0f743aa6fdead3eef59decfefc9de91d550bf0805b8fce16ed10d421ba"},
"open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},
"p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"},
"parse_trans": {:git, "https://github.com/uwiger/parse_trans.git", "76abb347c3c1d00fb0ccf9e4b43e22b3d2288484", [tag: "3.3.0"]},