From 0ff7f6d03ce5978b31374b18bfea32e4ec83704f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 29 Oct 2022 15:01:17 +0200 Subject: [PATCH 01/35] Do not use contentMap if content is not empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b2ba0cbbc8..f5dca24968 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Workers.TransmogrifierWorker import Ecto.Query + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] require Logger require Pleroma.Constants @@ -389,6 +390,8 @@ def fix_tag(%{"tag" => %{} = tag} = object) do def fix_tag(object), do: object + def fix_content_map(%{"content" => content} = object) when not_empty_string(content), do: object + # content map usually only has one language so this will do for now. def fix_content_map(%{"contentMap" => content_map} = object) do content_groups = Map.to_list(content_map) From ff35b13b5285f57217b85de1639414323b545cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 29 Oct 2022 16:12:37 +0200 Subject: [PATCH 02/35] Allow to specify post language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/constants.ex | 6 ++-- .../web/activity_pub/transmogrifier.ex | 20 +++++++---- lib/pleroma/web/activity_pub/utils.ex | 11 ++++-- .../web/activity_pub/views/object_view.ex | 6 ++-- lib/pleroma/web/common_api/activity_draft.ex | 24 +++++++++++++ .../transmogrifier/note_handling_test.exs | 30 ++++++++++++++++ .../web/activity_pub/transmogrifier_test.exs | 12 +++++++ test/pleroma/web/activity_pub/utils_test.exs | 34 +++++++++++++------ 8 files changed, 120 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index f848e9fb58..603218498e 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -22,7 +22,8 @@ defmodule Pleroma.Constants do "generator", "assigned_account", "rules", - "content_type" + "content_type", + "language" ] ) @@ -41,7 +42,8 @@ defmodule Pleroma.Constants do "summary", "sensitive", "attachment", - "generator" + "generator", + "language" ] ) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f5dca24968..abcf7d63b0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -772,6 +772,7 @@ def prepare_object(object) do |> add_mention_tags |> add_emoji_tags |> add_attributed_to + |> maybe_add_content_map |> prepare_attachments |> set_conversation |> set_reply_to_uri @@ -817,7 +818,7 @@ def prepare_outgoing(%{"type" => activity_type, "object" => object_id} = data) data = data |> Map.put("object", object) - |> Map.merge(Utils.make_json_ld_header()) + |> Map.merge(Utils.make_json_ld_header(data)) |> Map.delete("bcc") {:ok, data} @@ -832,7 +833,7 @@ def prepare_outgoing(%{"type" => "Update", "object" => %{"type" => objtype} = ob data = data |> Map.put("object", object) - |> Map.merge(Utils.make_json_ld_header()) + |> Map.merge(Utils.make_json_ld_header(data)) |> Map.delete("bcc") {:ok, data} @@ -853,7 +854,7 @@ def prepare_outgoing(%{"type" => "Announce", "actor" => ap_id, "object" => objec data = data |> strip_internal_fields - |> Map.merge(Utils.make_json_ld_header()) + |> Map.merge(Utils.make_json_ld_header(data)) |> Map.delete("bcc") {:ok, data} @@ -873,7 +874,7 @@ def prepare_outgoing(%{"type" => "Accept"} = data) do data = data |> Map.put("object", object) - |> Map.merge(Utils.make_json_ld_header()) + |> Map.merge(Utils.make_json_ld_header(data)) {:ok, data} end @@ -891,7 +892,7 @@ def prepare_outgoing(%{"type" => "Reject"} = data) do data = data |> Map.put("object", object) - |> Map.merge(Utils.make_json_ld_header()) + |> Map.merge(Utils.make_json_ld_header(data)) {:ok, data} end @@ -902,7 +903,7 @@ def prepare_outgoing(%{"type" => _type} = data) do data |> strip_internal_fields |> maybe_fix_object_url - |> Map.merge(Utils.make_json_ld_header()) + |> Map.merge(Utils.make_json_ld_header(data)) {:ok, data} end @@ -1088,4 +1089,11 @@ def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do def maybe_fix_user_url(data), do: data def maybe_fix_user_object(data), do: maybe_fix_user_url(data) + + defp maybe_add_content_map(%{"language" => language, "content" => content} = object) + when not_empty_string(language) do + Map.put(object, "contentMap", Map.put(%{}, language, content)) + end + + defp maybe_add_content_map(object), do: object end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index ffac168751..696971d3f8 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Pleroma.Web.Router.Helpers import Ecto.Query + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] require Logger require Pleroma.Constants @@ -107,18 +108,24 @@ def maybe_splice_recipient(ap_id, params) do end end - def make_json_ld_header do + def make_json_ld_header(data \\ %{}) do %{ "@context" => [ "https://www.w3.org/ns/activitystreams", "#{Endpoint.url()}/schemas/litepub-0.1.jsonld", %{ - "@language" => "und" + "@language" => get_language(data) } ] } end + defp get_language(%{"language" => language}) when not_empty_string(language) do + language + end + + defp get_language(_), do: "und" + def make_date do DateTime.utc_now() |> DateTime.to_iso8601() end diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index 63caa915c0..13b5b25429 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do alias Pleroma.Web.ActivityPub.Transmogrifier def render("object.json", %{object: %Object{} = object}) do - base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header() + base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(object.data) additional = Transmogrifier.prepare_object(object.data) Map.merge(base, additional) @@ -17,7 +17,7 @@ def render("object.json", %{object: %Object{} = object}) do def render("object.json", %{object: %Activity{data: %{"type" => activity_type}} = activity}) when activity_type in ["Create", "Listen"] do - base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header() + base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data) object = Object.normalize(activity, fetch: false) additional = @@ -28,7 +28,7 @@ def render("object.json", %{object: %Activity{data: %{"type" => activity_type}} end def render("object.json", %{object: %Activity{} = activity}) do - base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header() + base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data) object_id = Object.normalize(activity, id_only: true) additional = diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 3e68bcad60..8e2a6bedfd 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -14,6 +14,15 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do import Pleroma.Web.Gettext import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + @supported_locales ~w( + aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce ch co cr cs cu cv cy da + de dv dz ee el en eo es et eu fa ff fi fj fo fr fy ga gd gl gu gv ha he hi ho hr ht hu hy hz ia + id ie ig ii ik io is it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln lo + lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv ny oc oj om or os pa pi pl + ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl sn so sq sr ss st su sv sw ta te tg th ti tk tl + tn to tr ts tt tw ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu ast ckb kab kmr zgh + ) + defstruct valid?: true, errors: [], user: nil, @@ -36,6 +45,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do cc: [], context: nil, sensitive: false, + language: nil, object: nil, preview?: false, changes: %{} @@ -62,6 +72,7 @@ def create(user, params) do |> content() |> with_valid(&to_and_cc/1) |> with_valid(&context/1) + |> with_valid(&language/1) |> sensitive() |> with_valid(&object/1) |> preview?() @@ -224,6 +235,18 @@ defp sensitive(draft) do %__MODULE__{draft | sensitive: sensitive} end + defp language(%{params: %{language: language}} = draft) when not_empty_string(language) do + case language |> String.split("_") |> Enum.at(0) do + locale when locale in @supported_locales -> + %__MODULE__{draft | language: locale} + + _ -> + draft + end + end + + defp language(draft), do: draft + defp object(draft) do emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji) @@ -264,6 +287,7 @@ defp object(draft) do }) |> Map.put("generator", draft.params[:generator]) |> Map.put("content_type", draft.params[:content_type]) + |> Map.put("language", draft.language) %__MODULE__{draft | object: object} end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 7c406fbd05..c9d46ca2c6 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -220,6 +220,36 @@ test "it works for incoming notices with contentMap" do "

@lain

" end + test "it only uses contentMap if content is not present" do + user = insert(:user) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Create", + "object" => %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "id" => Utils.generate_object_id(), + "type" => "Note", + "content" => "Hi", + "contentMap" => %{ + "de" => "Hallo", + "uk" => "Привіт" + }, + "inReplyTo" => nil, + "attributedTo" => user.ap_id + }, + "actor" => user.ap_id + } + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message) + object = Object.normalize(data["object"], fetch: false) + + assert object.data["content"] == "Hi" + end + test "it works for incoming notices with to/cc not being an array (kroeg)" do data = File.read!("test/fixtures/kroeg-post-activity.json") |> Jason.decode!() diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 3044a75ff8..ce08b5ced1 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -392,6 +392,18 @@ test "it prepares a quote post" do assert modified["object"]["quoteUrl"] == quote_id assert modified["object"]["quoteUri"] == quote_id end + + test "it adds contentMap if language is specified" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "тест", language: "uk"}) + + {:ok, prepared} = Transmogrifier.prepare_outgoing(activity.data) + + assert prepared["object"]["contentMap"] == %{ + "uk" => "тест" + } + end end describe "user upgrade" do diff --git a/test/pleroma/web/activity_pub/utils_test.exs b/test/pleroma/web/activity_pub/utils_test.exs index 2904678c15..cd545bfaf6 100644 --- a/test/pleroma/web/activity_pub/utils_test.exs +++ b/test/pleroma/web/activity_pub/utils_test.exs @@ -138,16 +138,30 @@ test "does not adress actor's follower address if the activity is not public", % end end - test "make_json_ld_header/0" do - assert Utils.make_json_ld_header() == %{ - "@context" => [ - "https://www.w3.org/ns/activitystreams", - "http://localhost:4001/schemas/litepub-0.1.jsonld", - %{ - "@language" => "und" - } - ] - } + describe "make_json_ld_header/1" do + test "makes jsonld header" do + assert Utils.make_json_ld_header() == %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + %{ + "@language" => "und" + } + ] + } + end + + test "includes language if specified" do + assert Utils.make_json_ld_header(%{"language" => "pl"}) == %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + %{ + "@language" => "pl" + } + ] + } + end end describe "get_existing_votes" do From 6a8a8f2a2e2a34713bf495f58f84ee49f1b2a09b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 29 Oct 2022 19:49:52 +0200 Subject: [PATCH 03/35] StatusView: display language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/status_view.ex | 4 ++-- .../web/mastodon_api/views/status_view_test.exs | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5fdb614a37..c57d5346b2 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -225,7 +225,7 @@ def render( mentions: mentions, tags: reblogged[:tags] || [], application: build_application(object.data["generator"]), - language: nil, + language: object.data["language"], emojis: [], pleroma: %{ local: activity.local, @@ -428,7 +428,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} mentions: mentions, tags: build_tags(tags), application: build_application(object.data["generator"]), - language: nil, + language: object.data["language"], emojis: build_emojis(object.data["emoji"]), pleroma: %{ local: activity.local, diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 8dbb033859..5dafc0ff65 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -800,6 +800,16 @@ test "it shows edited_at" do assert status.edited_at end + test "it shows post language" do + user = insert(:user) + + {:ok, post} = CommonAPI.post(user, %{status: "Szczęść Boże", language: "pl"}) + + status = StatusView.render("show.json", activity: post) + + assert status.language == "pl" + end + test "with a source object" do note = insert(:note, From 972498cac453da859ccfe13eb6679be7ae22eab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 29 Oct 2022 22:19:47 +0200 Subject: [PATCH 04/35] Store remote post language information MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../article_note_page_validator.ex | 1 + .../object_validators/common_fields.ex | 1 + .../web/activity_pub/transmogrifier.ex | 37 +++++++++++++++++++ lib/pleroma/web/common_api/activity_draft.ex | 15 ++------ lib/pleroma/web/common_api/utils.ex | 18 +++++++++ 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 3a260f6e67..9a569bef8d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -108,6 +108,7 @@ defp fix(data) do |> fix_attachments() |> Transmogrifier.fix_emoji() |> Transmogrifier.fix_content_map() + |> Transmogrifier.maybe_add_language() end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index a625e76caa..3ad0503428 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -58,6 +58,7 @@ defmacro status_object_fields do field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) field(:quotes_count, :integer, default: 0) + field(:language, :string) field(:inReplyTo, ObjectValidators.ObjectID) field(:quoteUrl, ObjectValidators.ObjectID) field(:url, ObjectValidators.Uri) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index abcf7d63b0..5322b0a1ec 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -44,6 +44,7 @@ def fix_object(object, options \\ []) do |> fix_content_map() |> fix_addressing() |> fix_summary() + |> maybe_add_language() end def fix_summary(%{"summary" => nil} = object) do @@ -1096,4 +1097,40 @@ defp maybe_add_content_map(%{"language" => language, "content" => content} = obj end defp maybe_add_content_map(object), do: object + + def maybe_add_language(object) do + language = + get_language_from_context(object) |> Pleroma.Web.CommonAPI.Utils.get_valid_language() || + get_language_from_content_map(object) |> Pleroma.Web.CommonAPI.Utils.get_valid_language() + + if language do + Map.put(object, "language", language) + else + object + end + end + + defp get_language_from_context(%{"@context" => context}) do + case context + |> Enum.find(fn + %{"@language" => language} -> language != "und" + _ -> nil + end) do + %{"@language" => language} -> language + _ -> nil + end + end + + defp get_language_from_context(_), do: nil + + defp get_language_from_content_map(%{"contentMap" => content_map, "content" => source_content}) do + content_groups = Map.to_list(content_map) + + case Enum.find(content_groups, fn {_, content} -> content == source_content end) do + {language, _} -> language + _ -> nil + end + end + + defp get_language_from_content_map(_), do: nil end diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 8e2a6bedfd..9d4283b7fd 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -14,15 +14,6 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do import Pleroma.Web.Gettext import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] - @supported_locales ~w( - aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce ch co cr cs cu cv cy da - de dv dz ee el en eo es et eu fa ff fi fj fo fr fy ga gd gl gu gv ha he hi ho hr ht hu hy hz ia - id ie ig ii ik io is it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln lo - lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv ny oc oj om or os pa pi pl - ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl sn so sq sr ss st su sv sw ta te tg th ti tk tl - tn to tr ts tt tw ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu ast ckb kab kmr zgh - ) - defstruct valid?: true, errors: [], user: nil, @@ -236,9 +227,9 @@ defp sensitive(draft) do end defp language(%{params: %{language: language}} = draft) when not_empty_string(language) do - case language |> String.split("_") |> Enum.at(0) do - locale when locale in @supported_locales -> - %__MODULE__{draft | language: locale} + case Utils.get_valid_language(language) do + language when is_binary(language) -> + %__MODULE__{draft | language: language} _ -> draft diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index ff08143295..1af1b19839 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -23,6 +23,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do require Logger require Pleroma.Constants + @supported_locales ~w( + aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce ch co cr cs cu cv cy da + de dv dz ee el en eo es et eu fa ff fi fj fo fr fy ga gd gl gu gv ha he hi ho hr ht hu hy hz ia + id ie ig ii ik io is it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln lo + lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv ny oc oj om or os pa pi pl + ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl sn so sq sr ss st su sv sw ta te tg th ti tk tl + tn to tr ts tt tw ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu ast ckb kab kmr zgh + ) + def attachments_from_ids(%{media_ids: ids, descriptions: desc}) do attachments_from_ids_descs(ids, desc) end @@ -482,4 +491,13 @@ def validate_attachments_count(attachments) do {:error, dgettext("errors", "Too many attachments")} end end + + def get_valid_language(language) when is_binary(language) do + case language |> String.split("_") |> Enum.at(0) do + locale when locale in @supported_locales -> locale + _ -> nil + end + end + + def get_valid_language(_), do: nil end From 8243be681b06ee4d58d019463ad9bb397ef00891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 18:47:41 +0100 Subject: [PATCH 05/35] WIP Translation backends support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/application.ex | 3 +- .../api_spec/operations/status_operation.ex | 60 +++++++++++++++++++ .../controllers/status_controller.ex | 29 ++++++++- .../web/mastodon_api/views/status_view.ex | 8 +++ lib/pleroma/web/router.ex | 1 + 5 files changed, 99 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 57bc9618ee..c887671a06 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -212,7 +212,8 @@ defp cachex_children do build_cachex("chat_message_id_idempotency_key", expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 - ) + ), + build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000) ] end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 3d68ddc342..f235d8c47a 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -409,6 +409,38 @@ def context_operation do } end + def translate_operation do + %Operation{ + tags: ["Retrieve status information"], + summary: "Translate status", + description: "Translate status with an external API", + operationId: "StatusController.translate", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [id_param()], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + target_language: %Schema{ + type: :string, + nullable: true, + description: "Translation target language." + } + } + }, + required: false + ), + responses: %{ + 200 => Operation.response("Translation", "application/json", translation()) + 400 => Operation.response("Error", "application/json", ApiError) + 404 => Operation.response("Error", "application/json", ApiError) + 503 => Operation.response("Error", "application/json", ApiError) + } + } + end + def favourites_operation do %Operation{ tags: ["Timelines"], @@ -793,4 +825,32 @@ defp context do } } end + + defp translation do + %Schema{ + title: "StatusTranslation", + description: "Represents status translation with related information.", + type: :object, + required: [:content, :detected_source_language, :provider], + properties: %{ + content: %Schema{ + type: :string, + description: "Translated status content" + }, + detected_source_language: %Schema{ + type: :string, + description: "Detected source language" + }, + provider: %Schema{ + type: :string, + description: "Translation provider service name" + } + }, + example: %{ + "content" => "Software für die nächste Generation der sozialen Medien.", + "detected_source_language" => "en", + "provider" => "Deepl" + } + } + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 924e37aa7d..4dbfcd2621 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity + alias Pleroma.Translation alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility @@ -44,6 +45,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do ] ) + plug(OAuthScopesPlug, %{scopes: ["read:statuses"]} when action == :translate) + plug( OAuthScopesPlug, %{scopes: ["write:statuses"]} @@ -85,7 +88,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do %{scopes: ["write:bookmarks"]} when action in [:bookmark, :unbookmark] ) - @rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete)a + @rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete translate)a plug( RateLimiter, @@ -450,6 +453,30 @@ def context(%{assigns: %{user: user}} = conn, %{id: id}) do end end + @doc "POST /api/v1/statuses/:id/translate" + def translate(%{body_params: params, assigns: %{user: user}} = conn, %{id: status_id}) do + with %Activity{object: object} <- Activity.get_by_id_with_object(status_id), + {:language, language} when is_binary(language) <- + {:language, Map.get(params, :target_language) || user.language}, + {:ok, result} <- + Translation.translate( + object.data["content"], + object.data["language"], + language + ) do + render(conn, "translation.json", result) + else + {:language, nil} -> + render_error(conn, :bad_request, "Language not specified") + + {:error, :not_found} -> + render_error(conn, :not_found, "Translation service not configured") + + {:error, error} when error in [:unexpected_response, :quota_exceeded, :too_many_requests] -> + render_error(conn, :service_unavailable, "Translation service not available") + end + end + @doc "GET /api/v1/favourites" def favourites(%{assigns: %{user: %User{} = user}} = conn, params) do activities = ActivityPub.fetch_favourites(user, params) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c57d5346b2..0a49478e80 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -665,6 +665,14 @@ def render("context.json", %{activity: activity, activities: activities, user: u } end + def render("translation.json", %{ + content: content, + detected_source_language: detected_source_language, + provider: provider + }) do + %{content: content, detected_source_language: detected_source_language, provider: provider} + end + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity, fetch: false) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 065b136908..ba6642d4b0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -618,6 +618,7 @@ defmodule Pleroma.Web.Router do post("/statuses/:id/unbookmark", StatusController, :unbookmark) post("/statuses/:id/mute", StatusController, :mute_conversation) post("/statuses/:id/unmute", StatusController, :unmute_conversation) + post("/statuses/:id/translate", StatusController, :translate) post("/push/subscription", SubscriptionController, :create) get("/push/subscription", SubscriptionController, :show) From 25a9ff0f3dbc89ca004f26a8e214e98ba9b444ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 18:52:26 +0100 Subject: [PATCH 06/35] Expose translation service availability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 13 ++++ lib/pleroma/translation.ex | 54 +++++++++++++ lib/pleroma/translation/deepl.ex | 75 +++++++++++++++++++ lib/pleroma/translation/libretranslate.ex | 66 ++++++++++++++++ lib/pleroma/translation/service.ex | 20 +++++ .../api_spec/operations/status_operation.ex | 6 +- .../web/mastodon_api/views/instance_view.ex | 2 +- 7 files changed, 232 insertions(+), 4 deletions(-) create mode 100644 lib/pleroma/translation.ex create mode 100644 lib/pleroma/translation/deepl.ex create mode 100644 lib/pleroma/translation/libretranslate.ex create mode 100644 lib/pleroma/translation/service.ex diff --git a/config/description.exs b/config/description.exs index bdf57dd633..1598973df3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3550,5 +3550,18 @@ description: "Update nickname according to host-meta, when refetching the user" } ] + }, + %{ + group: :pleroma, + key: Pleroma.Translation, + type: :group, + description: "Translation providers", + children: [ + %{ + key: Pleroma.Translation, + type: :service, + suggestions: [Pleroma.Translation.DeepL, Pleroma.Translation.LibreTranslate] + } + ] } ] diff --git a/lib/pleroma/translation.ex b/lib/pleroma/translation.ex new file mode 100644 index 0000000000..112f12cab2 --- /dev/null +++ b/lib/pleroma/translation.ex @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation do + @cache_ttl 86_400_000 + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + + def configured? do + service = get_service() + + !!service and service.configured? + end + + def translate(text, source_language, target_language) do + cache_key = get_cache_key(text, source_language, target_language) + + case @cachex.get(:translations_cache, cache_key) do + {:ok, nil} -> + service = get_service() + + result = + if !service or !service.configured? do + {:error, :not_found} + else + service.translate(text, source_language, target_language) + end + + store_result(result, cache_key) + + result + + {:ok, result} -> + {:ok, result} + + {:error, error} -> + {:error, error} + end + end + + defp get_service, do: Pleroma.Config.get([__MODULE__, :service]) + + defp get_cache_key(text, source_language, target_language) do + "#{source_language}/#{target_language}/#{content_hash(text)}" + end + + defp store_result({:ok, result}, cache_key) do + @cachex.put(:translations_cache, cache_key, result, ttl: @cache_ttl) + end + + defp store_result(_, _), do: nil + + defp content_hash(text), do: :crypto.hash(:sha256, text) |> Base.encode64() +end diff --git a/lib/pleroma/translation/deepl.ex b/lib/pleroma/translation/deepl.ex new file mode 100644 index 0000000000..76fff46931 --- /dev/null +++ b/lib/pleroma/translation/deepl.ex @@ -0,0 +1,75 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation.DeepL do + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + + alias Pleroma.Translation.Service + + @behaviour Service + + @impl Service + def configured? do + not_empty_string(get_plan()) and not_empty_string(get_api_key()) + end + + @impl Service + def translate(content, source_language, target_language) do + endpoint = endpoint_url() + + case Pleroma.HTTP.post( + endpoint <> + "?" <> + URI.encode_query(%{ + text: content, + source_lang: source_language |> String.upcase(), + target_lang: target_language, + tag_handling: "html" + }), + "", + [ + {"Content-Type", "application/x-www-form-urlencoded"}, + {"Authorization", "DeepL-Auth-Key #{get_api_key()}"} + ] + ) do + {:ok, %{status: 429}} -> + {:error, :too_many_requests} + + {:ok, %{status: 456}} -> + {:error, :quota_exceeded} + + {:ok, %{status: 200} = res} -> + %{ + "translations" => [ + %{"text" => content, "detected_source_language" => detected_source_language} + ] + } = Jason.decode!(res.body) + + {:ok, + %{ + content: content, + detected_source_language: detected_source_language, + provider: "DeepL" + }} + + _ -> + {:error, :internal_server_error} + end + end + + defp endpoint_url do + case get_plan() do + "free" -> "https://api-free.deepl.com/v2/translate" + _ -> "https://api.deepl.com/v2/translate" + end + end + + defp get_plan do + Pleroma.Config.get([__MODULE__, :plan]) + end + + defp get_api_key do + Pleroma.Config.get([__MODULE__, :api_key]) + end +end diff --git a/lib/pleroma/translation/libretranslate.ex b/lib/pleroma/translation/libretranslate.ex new file mode 100644 index 0000000000..049053d430 --- /dev/null +++ b/lib/pleroma/translation/libretranslate.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation.LibreTranslate do + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + + alias Pleroma.Translation.Service + + @behaviour Service + + @impl Service + def configured?, do: not_empty_string(get_base_url()) + + @impl Service + def translate(content, source_language, target_language) do + endpoint = endpoint_url() + + case Pleroma.HTTP.post( + endpoint, + Jason.encode!(%{ + q: content, + source: source_language |> String.upcase(), + target: target_language, + format: "html", + api_key: get_api_key() + }), + [ + {"Content-Type", "application/json"} + ] + ) do + {:ok, %{status: 429}} -> + {:error, :too_many_requests} + + {:ok, %{status: 403}} -> + {:error, :quota_exceeded} + + {:ok, %{status: 200} = res} -> + %{ + "translatedText" => content + } = Jason.decode!(res.body) + + {:ok, + %{ + content: content, + detected_source_language: source_language, + provider: "LibreTranslate" + }} + + _ -> + {:error, :internal_server_error} + end + end + + defp endpoint_url do + get_base_url() <> "/translate" + end + + defp get_base_url do + Pleroma.Config.get([__MODULE__, :base_url]) + end + + defp get_api_key do + Pleroma.Config.get([__MODULE__, :api_key], "") + end +end diff --git a/lib/pleroma/translation/service.ex b/lib/pleroma/translation/service.ex new file mode 100644 index 0000000000..55e995e926 --- /dev/null +++ b/lib/pleroma/translation/service.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Translation.Service do + @callback configured?() :: boolean() + + @callback translate( + content :: String.t(), + source_language :: String.t(), + target_language :: String.t() + ) :: + {:ok, + %{ + content: String.t(), + detected_source_language: String.t(), + provider: String.t() + }} + | {:error, atom()} +end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index f235d8c47a..f93b50b573 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -433,9 +433,9 @@ def translate_operation do required: false ), responses: %{ - 200 => Operation.response("Translation", "application/json", translation()) - 400 => Operation.response("Error", "application/json", ApiError) - 404 => Operation.response("Error", "application/json", ApiError) + 200 => Operation.response("Translation", "application/json", translation()), + 400 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError), 503 => Operation.response("Error", "application/json", ApiError) } } diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f8852e9a45..71ae8322ee 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -206,7 +206,7 @@ def configuration2 do configuration() |> Map.merge(%{ urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()}, - translation: %{enabled: false} + translation: %{enabled: Pleroma.Translation.configured?} }) end From c1f5138f73cc253e0f37a1a84a467bb6623d9c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 21:06:31 +0100 Subject: [PATCH 07/35] Add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/translation.ex | 3 +- .../web/mastodon_api/views/instance_view.ex | 7 +++- test/pleroma/translation_test.ex | 29 ++++++++++++++ .../controllers/status_controller_test.exs | 40 +++++++++++++++++++ test/support/translation_mock.ex | 22 ++++++++++ 5 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 test/pleroma/translation_test.ex create mode 100644 test/support/translation_mock.ex diff --git a/lib/pleroma/translation.ex b/lib/pleroma/translation.ex index 112f12cab2..7efec62a62 100644 --- a/lib/pleroma/translation.ex +++ b/lib/pleroma/translation.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Translation do - @cache_ttl 86_400_000 @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def configured? do @@ -45,7 +44,7 @@ defp get_cache_key(text, source_language, target_language) do end defp store_result({:ok, result}, cache_key) do - @cachex.put(:translations_cache, cache_key, result, ttl: @cache_ttl) + @cachex.put(:translations_cache, cache_key, result) end defp store_result(_, _), do: nil diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 71ae8322ee..b3c12fec9a 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -142,7 +142,10 @@ def features do if Config.get([:instance, :profile_directory]) do "profile_directory" end, - "pleroma:get:main/ostatus" + "pleroma:get:main/ostatus", + if Pleroma.Translation.configured?() do + "translation" + end ] |> Enum.filter(& &1) end @@ -206,7 +209,7 @@ def configuration2 do configuration() |> Map.merge(%{ urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()}, - translation: %{enabled: Pleroma.Translation.configured?} + translation: %{enabled: Pleroma.Translation.configured?()} }) end diff --git a/test/pleroma/translation_test.ex b/test/pleroma/translation_test.ex new file mode 100644 index 0000000000..2ae7856ee4 --- /dev/null +++ b/test/pleroma/translation_test.ex @@ -0,0 +1,29 @@ +defmodule Pleroma.TranslationTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Translation + # use Oban.Testing, repo: Pleroma.Repo + + setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + + test "it translates text" do + assert {:ok, + %{ + content: "txet emos", + detected_source_language: _, + provider: _ + }} = Translation.translate("some text", "en", "uk") + end + + test "it stores translation result in cache" do + Translation.translate("some text", "en", "uk") + + assert {:ok, result} = + Cachex.get( + :translations_cache, + "en/uk/#{:crypto.hash(:sha256, "some text") |> Base.encode64()}" + ) + + assert result.content == "txet emos" + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 328e54c115..58bcd71776 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2212,4 +2212,44 @@ test "it returns 404 if the user cannot see the post", %{conn: conn} do |> json_response_and_validate_schema(:not_found) end end + + describe "translating statuses" do + setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + + test "it translates a status to user language" do + user = insert(:user, language: "fr") + %{conn: conn, user: user} = oauth_access(["read:statuses"], user: user) + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "Cześć!", + visibility: "public", + language: "pl" + }) + + response = + conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(200) + + assert response == %{"content" => "!ćśezC", "detected_source_language" => "pl", "provider" => "TranslationMock"} + end + + test "it returns an error if no target language provided" do + %{conn: conn, user: user} = oauth_access(["read:statuses"]) + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "Cześć!", + language: "pl" + }) + + response = + conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(400) + end + end end diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex new file mode 100644 index 0000000000..8da2116e82 --- /dev/null +++ b/test/support/translation_mock.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule TranslationMock do + alias Pleroma.Translation.Service + + @behaviour Service + + @impl Service + def configured?, do: true + + @impl Service + def translate(content, source_language, _target_language) do + {:ok, + %{ + content: content |> String.reverse(), + detected_source_language: source_language, + provider: "TranslationMock" + }} + end +end From 55cb9b29fea23dd7791841827d16b09830cb0a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 21:57:05 +0100 Subject: [PATCH 08/35] Do not translate non-public statuses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../controllers/status_controller.ex | 5 +++++ .../controllers/status_controller_test.exs | 22 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 4dbfcd2621..bcfdee1090 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -456,6 +456,8 @@ def context(%{assigns: %{user: user}} = conn, %{id: id}) do @doc "POST /api/v1/statuses/:id/translate" def translate(%{body_params: params, assigns: %{user: user}} = conn, %{id: status_id}) do with %Activity{object: object} <- Activity.get_by_id_with_object(status_id), + {:visibility, visibility} when visibility in ["public", "unlisted"] <- + {:visibility, Visibility.get_visibility(object)}, {:language, language} when is_binary(language) <- {:language, Map.get(params, :target_language) || user.language}, {:ok, result} <- @@ -469,6 +471,9 @@ def translate(%{body_params: params, assigns: %{user: user}} = conn, %{id: statu {:language, nil} -> render_error(conn, :bad_request, "Language not specified") + {:visibility, _} -> + render_error(conn, :not_found, "Record not found") + {:error, :not_found} -> render_error(conn, :not_found, "Translation service not configured") diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 58bcd71776..6d1c7befd7 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2233,7 +2233,11 @@ test "it translates a status to user language" do |> post("/api/v1/statuses/#{activity.id}/translate") |> json_response_and_validate_schema(200) - assert response == %{"content" => "!ćśezC", "detected_source_language" => "pl", "provider" => "TranslationMock"} + assert response == %{ + "content" => "!ćśezC", + "detected_source_language" => "pl", + "provider" => "TranslationMock" + } end test "it returns an error if no target language provided" do @@ -2251,5 +2255,21 @@ test "it returns an error if no target language provided" do |> post("/api/v1/statuses/#{activity.id}/translate") |> json_response_and_validate_schema(400) end + + test "it doesn't translate non-public statuses" do + %{conn: conn, user: user} = oauth_access(["read:statuses"]) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "Cześć!", + visibility: "private", + language: "pl" + }) + + response = + conn + |> post("/api/v1/statuses/#{activity.id}/translate") + |> json_response_and_validate_schema(404) + end end end From 68c00fa7496b86aa6145ec2598da64a35fa4b028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 21:59:48 +0100 Subject: [PATCH 09/35] Fix get_language_from_context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5322b0a1ec..2a0605fb9e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1110,7 +1110,7 @@ def maybe_add_language(object) do end end - defp get_language_from_context(%{"@context" => context}) do + defp get_language_from_context(%{"@context" => context}) when is_list(context) do case context |> Enum.find(fn %{"@language" => language} -> language != "und" From 9cdce15a42aaf6aba919bd7695978608f2b2da27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 30 Oct 2022 22:57:20 +0100 Subject: [PATCH 10/35] Update description.exs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 34 +++++++++++++++++++++-- lib/pleroma/translation/deepl.ex | 4 +-- lib/pleroma/translation/libretranslate.ex | 2 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/config/description.exs b/config/description.exs index 1598973df3..fed4f0ac99 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3558,9 +3558,37 @@ description: "Translation providers", children: [ %{ - key: Pleroma.Translation, - type: :service, - suggestions: [Pleroma.Translation.DeepL, Pleroma.Translation.LibreTranslate] + key: :service, + type: :module, + suggestions: [Pleroma.Translation.Deepl, Pleroma.Translation.Libretranslate] + }, + %{ + group: {:subgroup, Pleroma.Translation.Deepl}, + key: :plan, + label: "DeepL plan", + type: {:dropdown, :atom}, + suggestions: [:free, :pro] + }, + %{ + group: {:subgroup, Pleroma.Translation.Deepl}, + key: :api_key, + label: "DeepL API Key", + type: :string, + suggestions: ["YOUR_API_KEY"] + }, + %{ + group: {:subgroup, Pleroma.Translation.Libretranslate}, + key: :base_url, + label: "LibreTranslate plan", + type: :string, + suggestions: ["https://libretranslate.com"] + }, + %{ + group: {:subgroup, Pleroma.Translation.Libretranslate}, + key: :api_key, + label: "LibreTranslate API Key", + type: :string, + suggestions: ["YOUR_API_KEY"] } ] } diff --git a/lib/pleroma/translation/deepl.ex b/lib/pleroma/translation/deepl.ex index 76fff46931..944dab8ec8 100644 --- a/lib/pleroma/translation/deepl.ex +++ b/lib/pleroma/translation/deepl.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.DeepL do +defmodule Pleroma.Translation.Deepl do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] alias Pleroma.Translation.Service @@ -60,7 +60,7 @@ def translate(content, source_language, target_language) do defp endpoint_url do case get_plan() do - "free" -> "https://api-free.deepl.com/v2/translate" + :free -> "https://api-free.deepl.com/v2/translate" _ -> "https://api.deepl.com/v2/translate" end end diff --git a/lib/pleroma/translation/libretranslate.ex b/lib/pleroma/translation/libretranslate.ex index 049053d430..9c9b4b9b5d 100644 --- a/lib/pleroma/translation/libretranslate.ex +++ b/lib/pleroma/translation/libretranslate.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.LibreTranslate do +defmodule Pleroma.Translation.Libretranslate do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] alias Pleroma.Translation.Service From d094cdf55bcaf6becc27acce960e2ebbdba3ecbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 31 Oct 2022 21:58:10 +0100 Subject: [PATCH 11/35] Rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 17 ++++++++++------- lib/pleroma/{ => language}/translation.ex | 4 ++-- lib/pleroma/{ => language}/translation/deepl.ex | 10 +++++----- .../translation/libretranslate.ex | 10 +++++----- .../translation/provider.ex} | 2 +- .../controllers/status_controller.ex | 2 +- .../web/mastodon_api/views/instance_view.ex | 4 ++-- test/pleroma/{ => language}/translation_test.ex | 6 +++--- .../controllers/status_controller_test.exs | 2 +- test/support/translation_mock.ex | 8 ++++---- 10 files changed, 34 insertions(+), 31 deletions(-) rename lib/pleroma/{ => language}/translation.ex (92%) rename lib/pleroma/{ => language}/translation/deepl.ex (92%) rename lib/pleroma/{ => language}/translation/libretranslate.ex (90%) rename lib/pleroma/{translation/service.ex => language/translation/provider.ex} (92%) rename test/pleroma/{ => language}/translation_test.ex (79%) diff --git a/config/description.exs b/config/description.exs index fed4f0ac99..bf50ac1dab 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3553,38 +3553,41 @@ }, %{ group: :pleroma, - key: Pleroma.Translation, + key: Pleroma.Language.Translation, type: :group, description: "Translation providers", children: [ %{ - key: :service, + key: :provider, type: :module, - suggestions: [Pleroma.Translation.Deepl, Pleroma.Translation.Libretranslate] + suggestions: [ + Pleroma.Language.Translation.Deepl, + Pleroma.Language.Translation.Libretranslate + ] }, %{ - group: {:subgroup, Pleroma.Translation.Deepl}, + group: {:subgroup, Pleroma.Language.Translation.Deepl}, key: :plan, label: "DeepL plan", type: {:dropdown, :atom}, suggestions: [:free, :pro] }, %{ - group: {:subgroup, Pleroma.Translation.Deepl}, + group: {:subgroup, Pleroma.Language.Translation.Deepl}, key: :api_key, label: "DeepL API Key", type: :string, suggestions: ["YOUR_API_KEY"] }, %{ - group: {:subgroup, Pleroma.Translation.Libretranslate}, + group: {:subgroup, Pleroma.Language.Translation.Libretranslate}, key: :base_url, label: "LibreTranslate plan", type: :string, suggestions: ["https://libretranslate.com"] }, %{ - group: {:subgroup, Pleroma.Translation.Libretranslate}, + group: {:subgroup, Pleroma.Language.Translation.Libretranslate}, key: :api_key, label: "LibreTranslate API Key", type: :string, diff --git a/lib/pleroma/translation.ex b/lib/pleroma/language/translation.ex similarity index 92% rename from lib/pleroma/translation.ex rename to lib/pleroma/language/translation.ex index 7efec62a62..c9cd9d2dd5 100644 --- a/lib/pleroma/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation do +defmodule Pleroma.Language.Translation do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def configured? do @@ -37,7 +37,7 @@ def translate(text, source_language, target_language) do end end - defp get_service, do: Pleroma.Config.get([__MODULE__, :service]) + defp get_service, do: Pleroma.Config.get([__MODULE__, :provider]) defp get_cache_key(text, source_language, target_language) do "#{source_language}/#{target_language}/#{content_hash(text)}" diff --git a/lib/pleroma/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex similarity index 92% rename from lib/pleroma/translation/deepl.ex rename to lib/pleroma/language/translation/deepl.ex index 944dab8ec8..81048378c3 100644 --- a/lib/pleroma/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -2,19 +2,19 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.Deepl do +defmodule Pleroma.Language.Translation.Deepl do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] - alias Pleroma.Translation.Service + alias Pleroma.Language.Translation.Provider - @behaviour Service + @behaviour Provider - @impl Service + @impl Provider def configured? do not_empty_string(get_plan()) and not_empty_string(get_api_key()) end - @impl Service + @impl Provider def translate(content, source_language, target_language) do endpoint = endpoint_url() diff --git a/lib/pleroma/translation/libretranslate.ex b/lib/pleroma/language/translation/libretranslate.ex similarity index 90% rename from lib/pleroma/translation/libretranslate.ex rename to lib/pleroma/language/translation/libretranslate.ex index 9c9b4b9b5d..0c1fe17a04 100644 --- a/lib/pleroma/translation/libretranslate.ex +++ b/lib/pleroma/language/translation/libretranslate.ex @@ -2,17 +2,17 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.Libretranslate do +defmodule Pleroma.Language.Translation.Libretranslate do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] - alias Pleroma.Translation.Service + alias Pleroma.Language.Translation.Provider - @behaviour Service + @behaviour Provider - @impl Service + @impl Provider def configured?, do: not_empty_string(get_base_url()) - @impl Service + @impl Provider def translate(content, source_language, target_language) do endpoint = endpoint_url() diff --git a/lib/pleroma/translation/service.ex b/lib/pleroma/language/translation/provider.ex similarity index 92% rename from lib/pleroma/translation/service.ex rename to lib/pleroma/language/translation/provider.ex index 55e995e926..a88461a47f 100644 --- a/lib/pleroma/translation/service.ex +++ b/lib/pleroma/language/translation/provider.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Translation.Service do +defmodule Pleroma.Language.Translation.Provider do @callback configured?() :: boolean() @callback translate( diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index bcfdee1090..dc73cabf95 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -12,10 +12,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Language.Translation alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity - alias Pleroma.Translation alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index b3c12fec9a..91ebdae3c1 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -143,7 +143,7 @@ def features do "profile_directory" end, "pleroma:get:main/ostatus", - if Pleroma.Translation.configured?() do + if Pleroma.Language.Translation.configured?() do "translation" end ] @@ -209,7 +209,7 @@ def configuration2 do configuration() |> Map.merge(%{ urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()}, - translation: %{enabled: Pleroma.Translation.configured?()} + translation: %{enabled: Pleroma.Language.Translation.configured?()} }) end diff --git a/test/pleroma/translation_test.ex b/test/pleroma/language/translation_test.ex similarity index 79% rename from test/pleroma/translation_test.ex rename to test/pleroma/language/translation_test.ex index 2ae7856ee4..ecab3d20f7 100644 --- a/test/pleroma/translation_test.ex +++ b/test/pleroma/language/translation_test.ex @@ -1,10 +1,10 @@ -defmodule Pleroma.TranslationTest do +defmodule Pleroma.Language.TranslationTest do use Pleroma.Web.ConnCase - alias Pleroma.Translation + alias Pleroma.Language.Translation # use Oban.Testing, repo: Pleroma.Repo - setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) test "it translates text" do assert {:ok, diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 6d1c7befd7..d5dcaead65 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2214,7 +2214,7 @@ test "it returns 404 if the user cannot see the post", %{conn: conn} do end describe "translating statuses" do - setup do: clear_config([Pleroma.Translation, :service], TranslationMock) + setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) test "it translates a status to user language" do user = insert(:user, language: "fr") diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex index 8da2116e82..7e618c2639 100644 --- a/test/support/translation_mock.ex +++ b/test/support/translation_mock.ex @@ -3,14 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule TranslationMock do - alias Pleroma.Translation.Service + alias Pleroma.Language.Translation.Provider - @behaviour Service + @behaviour Provider - @impl Service + @impl Provider def configured?, do: true - @impl Service + @impl Provider def translate(content, source_language, _target_language) do {:ok, %{ From 4c1d0dbb69b055847d71eb4485bf27e89d9fab71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 3 Nov 2022 00:13:09 +0100 Subject: [PATCH 12/35] Language detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 22 +++++++++ lib/pleroma/application_requirements.ex | 22 ++++++++- lib/pleroma/language/language_detector.ex | 34 ++++++++++++++ .../language/language_detector/fasttext.ex | 47 +++++++++++++++++++ .../language/language_detector/provider.ex | 11 +++++ lib/pleroma/language/translation.ex | 12 ++--- lib/pleroma/web/common_api/activity_draft.ex | 15 +++--- 7 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 lib/pleroma/language/language_detector.ex create mode 100644 lib/pleroma/language/language_detector/fasttext.ex create mode 100644 lib/pleroma/language/language_detector/provider.ex diff --git a/config/description.exs b/config/description.exs index bf50ac1dab..f3395863c8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3594,5 +3594,27 @@ suggestions: ["YOUR_API_KEY"] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Language.LanguageDetector, + type: :group, + description: "Language detection providers", + children: [ + %{ + key: :provider, + type: :module, + suggestions: [ + Pleroma.Language.LanguageDetector.Fasttext + ] + }, + %{ + group: {:subgroup, Pleroma.Language.LanguageDetector.Fasttext}, + key: :model, + label: "fastText language detection model", + type: :string, + suggestions: ["/usr/share/fasttext/lid.176.bin"] + } + ] } ] diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 44b1c1705e..94d1ef7731 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -187,7 +187,27 @@ defp check_system_commands!(:ok) do false end - if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do + language_detector_commands_status = + if Pleroma.Language.LanguageDetector.missing_dependencies() == [] do + true + else + Logger.error( + "The following dependencies required by the currently enabled " <> + "language detection provider are not installed: " <> + inspect(Pleroma.Language.LanguageDetector.missing_dependencies()) + ) + + false + end + + if Enum.all?( + [ + preview_proxy_commands_status, + language_detector_commands_status + | filter_commands_statuses + ], + & &1 + ) do :ok else {:error, diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex new file mode 100644 index 0000000000..3901a8b90b --- /dev/null +++ b/lib/pleroma/language/language_detector.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetector do + @words_threshold 4 + + def missing_dependencies do + provider = get_provider() + + if provider do + provider.missing_dependencies() + else + nil + end + end + + def detect(text) do + provider = get_provider() + + {:ok, text} = text |> FastSanitize.strip_tags() + word_count = text |> String.split(~r/\s+/) |> Enum.count() + + if word_count < @words_threshold or !provider or !provider.configured? do + nil + else + provider.detect(text) + end + end + + defp get_provider() do + Pleroma.Config.get([__MODULE__, :provider]) + end +end diff --git a/lib/pleroma/language/language_detector/fasttext.ex b/lib/pleroma/language/language_detector/fasttext.ex new file mode 100644 index 0000000000..d479d21255 --- /dev/null +++ b/lib/pleroma/language/language_detector/fasttext.ex @@ -0,0 +1,47 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetector.Fasttext do + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + + alias Pleroma.Language.LanguageDetector.Provider + + @behaviour Provider + + @impl Provider + def missing_dependencies do + if Pleroma.Utils.command_available?("fasttext") do + [] + else + ["fasttext"] + end + end + + @impl Provider + def configured?, do: not_empty_string(get_model()) + + @impl Provider + def detect(text) do + text_path = Path.join(System.tmp_dir!(), "fasttext-#{Ecto.UUID.generate()}") + + File.write(text_path, text) + + detected_language = + case System.cmd("fasttext", ["predict", get_model(), text_path]) do + {"__label__" <> language, _} -> + language |> String.trim() + + _ -> + nil + end + + File.rm(text_path) + + detected_language + end + + defp get_model do + Pleroma.Config.get([__MODULE__, :model]) + end +end diff --git a/lib/pleroma/language/language_detector/provider.ex b/lib/pleroma/language/language_detector/provider.ex new file mode 100644 index 0000000000..08e7c8eef6 --- /dev/null +++ b/lib/pleroma/language/language_detector/provider.ex @@ -0,0 +1,11 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetector.Provider do + @callback missing_dependencies() :: [String.t()] + + @callback configured?() :: boolean() + + @callback detect(text :: String.t()) :: String.t() | nil +end diff --git a/lib/pleroma/language/translation.ex b/lib/pleroma/language/translation.ex index c9cd9d2dd5..c812385258 100644 --- a/lib/pleroma/language/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -6,9 +6,9 @@ defmodule Pleroma.Language.Translation do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def configured? do - service = get_service() + provider = get_provider() - !!service and service.configured? + !!provider and provider.configured? end def translate(text, source_language, target_language) do @@ -16,13 +16,13 @@ def translate(text, source_language, target_language) do case @cachex.get(:translations_cache, cache_key) do {:ok, nil} -> - service = get_service() + provider = get_provider() result = - if !service or !service.configured? do + if !configured?() do {:error, :not_found} else - service.translate(text, source_language, target_language) + provider.translate(text, source_language, target_language) end store_result(result, cache_key) @@ -37,7 +37,7 @@ def translate(text, source_language, target_language) do end end - defp get_service, do: Pleroma.Config.get([__MODULE__, :provider]) + defp get_provider, do: Pleroma.Config.get([__MODULE__, :provider]) defp get_cache_key(text, source_language, target_language) do "#{source_language}/#{target_language}/#{content_hash(text)}" diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 9d4283b7fd..91be325380 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do alias Pleroma.Activity alias Pleroma.Conversation.Participation + alias Pleroma.Language.LanguageDetector alias Pleroma.Object alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Visibility @@ -226,18 +227,14 @@ defp sensitive(draft) do %__MODULE__{draft | sensitive: sensitive} end - defp language(%{params: %{language: language}} = draft) when not_empty_string(language) do - case Utils.get_valid_language(language) do - language when is_binary(language) -> - %__MODULE__{draft | language: language} + defp language(draft) do + language = + Utils.get_valid_language(draft.params[:language]) || + LanguageDetector.detect(draft.full_payload) - _ -> - draft - end + %__MODULE__{draft | language: language} end - defp language(draft), do: draft - defp object(draft) do emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji) From 39d7093ff5fa9989762f2bbc02b05146882073db Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 3 Nov 2022 13:07:50 -0500 Subject: [PATCH 13/35] Remove Shout feature --- config/config.exs | 18 ----- config/description.exs | 21 ------ docs/configuration/cheatsheet.md | 5 -- .../API/differences_in_mastoapi_responses.md | 1 - docs/development/API/nodeinfo.md | 2 - lib/pleroma/application.ex | 12 --- lib/pleroma/config/deprecation_warnings.ex | 24 ------ lib/pleroma/config/transfer_task.ex | 1 - lib/pleroma/web/api_spec/schemas/account.ex | 3 - lib/pleroma/web/channels/user_socket.ex | 45 ------------ lib/pleroma/web/endpoint.ex | 1 - .../controllers/account_controller.ex | 5 +- .../web/mastodon_api/views/account_view.ex | 10 --- .../web/mastodon_api/views/instance_view.ex | 9 --- lib/pleroma/web/shout_channel.ex | 59 --------------- .../20200806175913_rename_instance_chat.exs | 73 +------------------ .../config/deprecation_warnings_test.exs | 10 --- test/pleroma/config/transfer_task_test.exs | 8 +- .../migrations/rename_instance_chat_test.exs | 56 -------------- .../controllers/config_controller_test.exs | 20 ++--- .../controllers/account_controller_test.exs | 1 - .../controllers/instance_controller_test.exs | 1 - test/pleroma/web/shout_channel_test.exs | 41 ----------- 23 files changed, 17 insertions(+), 409 deletions(-) delete mode 100644 lib/pleroma/web/channels/user_socket.ex delete mode 100644 lib/pleroma/web/shout_channel.ex delete mode 100644 test/pleroma/repo/migrations/rename_instance_chat_test.exs delete mode 100644 test/pleroma/web/shout_channel_test.exs diff --git a/config/config.exs b/config/config.exs index 7a621cbc2f..bb6ea2b2f9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -110,17 +110,6 @@ "xmpp" ] -websocket_config = [ - path: "/websocket", - serializer: [ - {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"}, - {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"} - ], - timeout: 60_000, - transport_log: false, - compress: false -] - # Configures the endpoint config :pleroma, Pleroma.Web.Endpoint, url: [host: "localhost"], @@ -130,9 +119,6 @@ {:_, [ {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, - {"/websocket", Phoenix.Endpoint.CowboyWebSocket, - {Phoenix.Transports.WebSocket, - {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} ]} ] @@ -475,10 +461,6 @@ image_quality: 85, min_content_length: 100 * 1024 -config :pleroma, :shout, - enabled: true, - limit: 5_000 - config :phoenix, :format_encoders, json: Jason, "activity+json": Jason config :phoenix, :json_library, Jason diff --git a/config/description.exs b/config/description.exs index bdf57dd633..afd87a2693 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2742,27 +2742,6 @@ } ] }, - %{ - group: :pleroma, - key: :shout, - type: :group, - description: "Pleroma shout settings", - children: [ - %{ - key: :enabled, - type: :boolean, - description: "Enables the backend Shoutbox chat feature." - }, - %{ - key: :limit, - type: :integer, - description: "Shout message character limit.", - suggestions: [ - 5_000 - ] - } - ] - }, %{ group: :pleroma, key: :http, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index f5b2a800f6..7971a12aba 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -8,11 +8,6 @@ For from source installations Pleroma configuration works by first importing the To add configuration to your config file, you can copy it from the base config. The latest version of it can be viewed [here](https://git.pleroma.social/pleroma/pleroma/blob/develop/config/config.exs). You can also use this file if you don't know how an option is supposed to be formatted. -## :shout - -* `enabled` - Enables the backend Shoutbox chat feature. Defaults to `true`. -* `limit` - Shout character limit. Defaults to `5_000` - ## :instance * `name`: The instance’s name. * `email`: Email used to reach an Administrator/Moderator of the instance. diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 4007c63c82..1de90d5cc0 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -103,7 +103,6 @@ Has these additional fields under the `pleroma` object: - `hide_followers_count`: boolean, true when the user has follower stat hiding enabled - `hide_follows_count`: boolean, true when the user has follow stat hiding enabled - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `/api/v1/accounts/verify_credentials` and `/api/v1/accounts/update_credentials` -- `chat_token`: The token needed for Pleroma shoutbox. Only returned in `/api/v1/accounts/verify_credentials` - `deactivated`: boolean, true when the user is deactivated - `allow_following_move`: boolean, true when the user allows automatically follow moved following accounts - `unread_conversation_count`: The count of unread conversations. Only returned to the account owner. diff --git a/docs/development/API/nodeinfo.md b/docs/development/API/nodeinfo.md index 0f998a1e6c..c76567feb6 100644 --- a/docs/development/API/nodeinfo.md +++ b/docs/development/API/nodeinfo.md @@ -45,7 +45,6 @@ See also [the Nodeinfo standard](https://nodeinfo.diaspora.software/). "multifetch", "pleroma:api/v1/notifications:include_types_filter", "chat", - "shout", "relay", "pleroma_emoji_reactions", "pleroma_chat_messages" @@ -205,7 +204,6 @@ See also [the Nodeinfo standard](https://nodeinfo.diaspora.software/). "multifetch", "pleroma:api/v1/notifications:include_types_filter", "chat", - "shout", "relay", "pleroma_emoji_reactions", "pleroma_chat_messages" diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 5e0b1ebfa9..5b1dc94365 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -112,7 +112,6 @@ def start(_type, _args) do ] ++ task_children(@mix_env) ++ dont_run_in_test(@mix_env) ++ - shout_child(shout_enabled?()) ++ [Pleroma.Gopher.Server] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html @@ -237,8 +236,6 @@ def build_cachex(type, opts), type: :worker } - defp shout_enabled?, do: Config.get([:shout, :enabled]) - defp dont_run_in_test(env) when env in [:test, :benchmark], do: [] defp dont_run_in_test(_) do @@ -259,15 +256,6 @@ defp background_migrators do ] end - defp shout_child(true) do - [ - Pleroma.Web.ShoutChannel.ShoutChannelState, - {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]} - ] - end - - defp shout_child(_), do: [] - defp task_children(:test) do [ %{ diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 599f1d3cfb..8f38046c4c 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -214,7 +214,6 @@ def warn do check_activity_expiration_config(), check_remote_ip_plug_name(), check_uploders_s3_public_endpoint(), - check_old_chat_shoutbox(), check_quarantined_instances_tuples(), check_transparency_exclusions_tuples(), check_simple_policy_tuples(), @@ -392,27 +391,4 @@ def check_uploders_s3_public_endpoint do :ok end end - - @spec check_old_chat_shoutbox() :: :ok | nil - def check_old_chat_shoutbox do - instance_config = Pleroma.Config.get([:instance]) - chat_config = Pleroma.Config.get([:chat]) || [] - - use_old_config = - Keyword.has_key?(instance_config, :chat_limit) or - Keyword.has_key?(chat_config, :enabled) - - if use_old_config do - Logger.error(""" - !!!DEPRECATION WARNING!!! - Your config is using the old namespace for the Shoutbox configuration. You need to convert to the new namespace. e.g., - \n* `config :pleroma, :chat, enabled` and `config :pleroma, :instance, chat_limit` are now equal to: - \n* `config :pleroma, :shout, enabled` and `config :pleroma, :shout, limit` - """) - - :error - else - :ok - end - end end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 4199630afe..ef8780e06b 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -16,7 +16,6 @@ defmodule Pleroma.Config.TransferTask do defp reboot_time_keys, do: [ {:pleroma, :hackney_pools}, - {:pleroma, :shout}, {:pleroma, Oban}, {:pleroma, :rate_limit}, {:pleroma, :markup}, diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 200d4bbb88..83731e1a02 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -49,7 +49,6 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do }, background_image: %Schema{type: :string, nullable: true, format: :uri}, birthday: %Schema{type: :string, nullable: true, format: :date}, - chat_token: %Schema{type: :string}, is_confirmed: %Schema{ type: :boolean, description: @@ -180,8 +179,6 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "is_moderator" => false, "skip_thread_containment" => false, "accepts_chat_messages" => true, - "chat_token" => - "SFMyNTY.g3QAAAACZAAEZGF0YW0AAAASOXRLaTNlc2JHN09RZ1oyOTIwZAAGc2lnbmVkbgYARNplS3EB.Mb_Iaqew2bN1I1o79B_iP7encmVCpTKC4OtHZRxdjKc", "unread_conversation_count" => 0, "tags" => [], "notification_settings" => %{ diff --git a/lib/pleroma/web/channels/user_socket.ex b/lib/pleroma/web/channels/user_socket.ex deleted file mode 100644 index 0f61b80c36..0000000000 --- a/lib/pleroma/web/channels/user_socket.ex +++ /dev/null @@ -1,45 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.UserSocket do - use Phoenix.Socket - alias Pleroma.User - - ## Channels - # channel "room:*", Pleroma.Web.RoomChannel - channel("chat:*", Pleroma.Web.ShoutChannel) - - # Socket params are passed from the client and can - # be used to verify and authenticate a user. After - # verification, you can put default assigns into - # the socket that will be set for all channels, ie - # - # {:ok, assign(socket, :user_id, verified_user_id)} - # - # To deny connection, return `:error`. - # - # See `Phoenix.Token` documentation for examples in - # performing token verification on connect. - def connect(%{"token" => token}, socket) do - with true <- Pleroma.Config.get([:shout, :enabled]), - {:ok, user_id} <- Phoenix.Token.verify(socket, "user socket", token, max_age: 84_600), - %User{} = user <- Pleroma.User.get_cached_by_id(user_id) do - {:ok, assign(socket, :user_name, user.nickname)} - else - _e -> :error - end - end - - # Socket id's are topics that allow you to identify all sockets for a given user: - # - # def id(socket), do: "user_socket:#{socket.assigns.user_id}" - # - # Would allow you to broadcast a "disconnect" event and terminate - # all active sockets and channels for a given user: - # - # Pleroma.Web.Endpoint.broadcast("user_socket:#{user.id}", "disconnect", %{}) - # - # Returning `nil` makes this socket anonymous. - def id(_socket), do: nil -end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index f311037227..e33b6527cf 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - socket("/socket", Pleroma.Web.UserSocket) socket("/live", Phoenix.LiveView.Socket) plug(Unplug, diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index d6072e36af..f54e0589e0 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -157,13 +157,10 @@ defp validate_email_param(_) do @doc "GET /api/v1/accounts/verify_credentials" def verify_credentials(%{assigns: %{user: user}} = conn, _) do - chat_token = Phoenix.Token.sign(conn, "user socket", user.id) - render(conn, "show.json", user: user, for: user, - with_pleroma_settings: true, - with_chat_token: chat_token + with_pleroma_settings: true ) end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 7a6c5c6fce..431fbf787c 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -305,7 +305,6 @@ defp do_render("show.json", %{user: user} = opts) do |> maybe_put_settings(user, opts[:for], opts) |> maybe_put_notification_settings(user, opts[:for]) |> maybe_put_settings_store(user, opts[:for], opts) - |> maybe_put_chat_token(user, opts[:for], opts) |> maybe_put_activation_status(user, opts[:for]) |> maybe_put_follow_requests_count(user, opts[:for]) |> maybe_put_allow_following_move(user, opts[:for]) @@ -362,15 +361,6 @@ defp maybe_put_settings_store(data, %User{} = user, %User{}, %{ defp maybe_put_settings_store(data, _, _, _), do: data - defp maybe_put_chat_token(data, %User{id: id}, %User{id: id}, %{ - with_chat_token: token - }) do - data - |> Kernel.put_in([:pleroma, :chat_token], token) - end - - defp maybe_put_chat_token(data, _, _, _), do: data - defp maybe_put_role(data, %User{show_role: true} = user, _) do data |> Kernel.put_in([:pleroma, :is_admin], user.is_admin) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f8852e9a45..63673ef17b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -41,7 +41,6 @@ def render("show.json", _) do background_upload_limit: Keyword.get(instance, :background_upload_limit), banner_upload_limit: Keyword.get(instance, :banner_upload_limit), background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), - shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), pleroma: pleroma_configuration(instance), soapbox: %{ @@ -120,13 +119,6 @@ def features do if Config.get([:gopher, :enabled]) do "gopher" end, - # backwards compat - if Config.get([:shout, :enabled]) do - "chat" - end, - if Config.get([:shout, :enabled]) do - "shout" - end, if Config.get([:instance, :allow_relay]) do "relay" end, @@ -245,7 +237,6 @@ defp pleroma_configuration2(instance) do banner_upload_limit: Keyword.get(instance, :banner_upload_limit), background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), - shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit) }) }) diff --git a/lib/pleroma/web/shout_channel.ex b/lib/pleroma/web/shout_channel.ex deleted file mode 100644 index 928f0a1dd0..0000000000 --- a/lib/pleroma/web/shout_channel.ex +++ /dev/null @@ -1,59 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ShoutChannel do - use Phoenix.Channel - - alias Pleroma.User - alias Pleroma.Web.MastodonAPI.AccountView - alias Pleroma.Web.ShoutChannel.ShoutChannelState - - def join("chat:public", _message, socket) do - send(self(), :after_join) - {:ok, socket} - end - - def handle_info(:after_join, socket) do - push(socket, "messages", %{messages: ShoutChannelState.messages()}) - {:noreply, socket} - end - - def handle_in("new_msg", %{"text" => text}, %{assigns: %{user_name: user_name}} = socket) do - text = String.trim(text) - - if String.length(text) in 1..Pleroma.Config.get([:shout, :limit]) do - author = User.get_cached_by_nickname(user_name) - author_json = AccountView.render("show.json", user: author, skip_visibility_check: true) - - message = ShoutChannelState.add_message(%{text: text, author: author_json}) - - broadcast!(socket, "new_msg", message) - end - - {:noreply, socket} - end -end - -defmodule Pleroma.Web.ShoutChannel.ShoutChannelState do - use Agent - - @max_messages 20 - - def start_link(_) do - Agent.start_link(fn -> %{max_id: 1, messages: []} end, name: __MODULE__) - end - - def add_message(message) do - Agent.get_and_update(__MODULE__, fn state -> - id = state[:max_id] + 1 - message = Map.put(message, "id", id) - messages = [message | state[:messages]] |> Enum.take(@max_messages) - {message, %{max_id: id, messages: messages}} - end) - end - - def messages do - Agent.get(__MODULE__, fn state -> state[:messages] |> Enum.reverse() end) - end -end diff --git a/priv/repo/migrations/20200806175913_rename_instance_chat.exs b/priv/repo/migrations/20200806175913_rename_instance_chat.exs index 47c568db7f..44d3530996 100644 --- a/priv/repo/migrations/20200806175913_rename_instance_chat.exs +++ b/priv/repo/migrations/20200806175913_rename_instance_chat.exs @@ -7,75 +7,6 @@ defmodule Pleroma.Repo.Migrations.RenameInstanceChat do alias Pleroma.ConfigDB - @instance_params %{group: :pleroma, key: :instance} - @shout_params %{group: :pleroma, key: :shout} - @chat_params %{group: :pleroma, key: :chat} - - def up do - instance_updated? = maybe_update_instance_key(:up) != :noop - chat_updated? = maybe_update_chat_key(:up) != :noop - - case Enum.any?([instance_updated?, chat_updated?]) do - true -> :ok - false -> :noop - end - end - - def down do - instance_updated? = maybe_update_instance_key(:down) != :noop - chat_updated? = maybe_update_chat_key(:down) != :noop - - case Enum.any?([instance_updated?, chat_updated?]) do - true -> :ok - false -> :noop - end - end - - # pleroma.instance.chat_limit -> pleroma.shout.limit - defp maybe_update_instance_key(:up) do - with %ConfigDB{value: values} <- ConfigDB.get_by_params(@instance_params), - limit when is_integer(limit) <- values[:chat_limit] do - @shout_params |> Map.put(:value, limit: limit) |> ConfigDB.update_or_create() - @instance_params |> Map.put(:subkeys, [":chat_limit"]) |> ConfigDB.delete() - else - _ -> - :noop - end - end - - # pleroma.shout.limit -> pleroma.instance.chat_limit - defp maybe_update_instance_key(:down) do - with %ConfigDB{value: values} <- ConfigDB.get_by_params(@shout_params), - limit when is_integer(limit) <- values[:limit] do - @instance_params |> Map.put(:value, chat_limit: limit) |> ConfigDB.update_or_create() - @shout_params |> Map.put(:subkeys, [":limit"]) |> ConfigDB.delete() - else - _ -> - :noop - end - end - - # pleroma.chat.enabled -> pleroma.shout.enabled - defp maybe_update_chat_key(:up) do - with %ConfigDB{value: values} <- ConfigDB.get_by_params(@chat_params), - enabled? when is_boolean(enabled?) <- values[:enabled] do - @shout_params |> Map.put(:value, enabled: enabled?) |> ConfigDB.update_or_create() - @chat_params |> Map.put(:subkeys, [":enabled"]) |> ConfigDB.delete() - else - _ -> - :noop - end - end - - # pleroma.shout.enabled -> pleroma.chat.enabled - defp maybe_update_chat_key(:down) do - with %ConfigDB{value: values} <- ConfigDB.get_by_params(@shout_params), - enabled? when is_boolean(enabled?) <- values[:enabled] do - @chat_params |> Map.put(:value, enabled: enabled?) |> ConfigDB.update_or_create() - @shout_params |> Map.put(:subkeys, [":enabled"]) |> ConfigDB.delete() - else - _ -> - :noop - end - end + def up, do: :noop + def down, do: :noop end diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index f3453ddb09..18497a1078 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -379,14 +379,4 @@ test "pool timeout" do "Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings" end end - - test "check_old_chat_shoutbox/0" do - clear_config([:instance, :chat_limit], 1_000) - clear_config([:chat, :enabled], true) - - assert capture_log(fn -> - DeprecationWarnings.check_old_chat_shoutbox() - end) =~ - "Your config is using the old namespace for the Shoutbox configuration." - end end diff --git a/test/pleroma/config/transfer_task_test.exs b/test/pleroma/config/transfer_task_test.exs index 3dc917362d..b1744dad36 100644 --- a/test/pleroma/config/transfer_task_test.exs +++ b/test/pleroma/config/transfer_task_test.exs @@ -110,8 +110,8 @@ test "don't restart if no reboot time settings were changed" do end test "on reboot time key" do - clear_config(:shout) - insert(:config, key: :shout, value: [enabled: false]) + clear_config([:rate_limit, :enabled], true) + insert(:config, key: :rate_limit, value: [enabled: false]) # Note that we don't actually restart Pleroma. # See module Restarter.Pleroma @@ -144,10 +144,10 @@ test "on reboot time subkey" do end test "don't restart pleroma on reboot time key and subkey if there is false flag" do - clear_config(:shout) + clear_config([:rate_limit, :enabled], true) clear_config(Pleroma.Captcha) - insert(:config, key: :shout, value: [enabled: false]) + insert(:config, key: :rate_limit, value: [enabled: false]) insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60]) refute String.contains?( diff --git a/test/pleroma/repo/migrations/rename_instance_chat_test.exs b/test/pleroma/repo/migrations/rename_instance_chat_test.exs deleted file mode 100644 index 17c39fd271..0000000000 --- a/test/pleroma/repo/migrations/rename_instance_chat_test.exs +++ /dev/null @@ -1,56 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Repo.Migrations.RenameInstanceChatTest do - use Pleroma.DataCase - import Pleroma.Factory - import Pleroma.Tests.Helpers - alias Pleroma.ConfigDB - - setup do: clear_config([:instance]) - setup do: clear_config([:chat]) - setup_all do: require_migration("20200806175913_rename_instance_chat") - - describe "up/0" do - test "migrates chat settings to shout", %{migration: migration} do - insert(:config, group: :pleroma, key: :instance, value: [chat_limit: 6000]) - insert(:config, group: :pleroma, key: :chat, value: [enabled: true]) - - assert migration.up() == :ok - - assert ConfigDB.get_by_params(%{group: :pleroma, key: :chat}) == nil - assert ConfigDB.get_by_params(%{group: :pleroma, key: :instance}) == nil - - assert ConfigDB.get_by_params(%{group: :pleroma, key: :shout}).value == [ - limit: 6000, - enabled: true - ] - end - - test "does nothing when chat settings are not set", %{migration: migration} do - assert migration.up() == :noop - assert ConfigDB.get_by_params(%{group: :pleroma, key: :chat}) == nil - assert ConfigDB.get_by_params(%{group: :pleroma, key: :shout}) == nil - end - end - - describe "down/0" do - test "migrates shout settings back to instance and chat", %{migration: migration} do - insert(:config, group: :pleroma, key: :shout, value: [limit: 42, enabled: true]) - - assert migration.down() == :ok - - assert ConfigDB.get_by_params(%{group: :pleroma, key: :chat}).value == [enabled: true] - assert ConfigDB.get_by_params(%{group: :pleroma, key: :instance}).value == [chat_limit: 42] - assert ConfigDB.get_by_params(%{group: :pleroma, key: :shout}) == nil - end - - test "does nothing when shout settings are not set", %{migration: migration} do - assert migration.down() == :noop - assert ConfigDB.get_by_params(%{group: :pleroma, key: :chat}) == nil - assert ConfigDB.get_by_params(%{group: :pleroma, key: :instance}) == nil - assert ConfigDB.get_by_params(%{group: :pleroma, key: :shout}) == nil - end - end -end diff --git a/test/pleroma/web/admin_api/controllers/config_controller_test.exs b/test/pleroma/web/admin_api/controllers/config_controller_test.exs index 6d014b65b1..050efa5e12 100644 --- a/test/pleroma/web/admin_api/controllers/config_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/config_controller_test.exs @@ -409,7 +409,7 @@ test "saving config with partial update", %{conn: conn} do end test "saving config which need pleroma reboot", %{conn: conn} do - clear_config([:shout, :enabled], true) + clear_config([:streamer, :workers], 3) assert conn |> put_req_header("content-type", "application/json") @@ -417,17 +417,17 @@ test "saving config which need pleroma reboot", %{conn: conn} do "/api/pleroma/admin/config", %{ configs: [ - %{group: ":pleroma", key: ":shout", value: [%{"tuple" => [":enabled", true]}]} + %{group: ":pleroma", key: ":streamer", value: [%{"tuple" => [":workers", 5]}]} ] } ) |> json_response_and_validate_schema(200) == %{ "configs" => [ %{ - "db" => [":enabled"], + "db" => [":workers"], "group" => ":pleroma", - "key" => ":shout", - "value" => [%{"tuple" => [":enabled", true]}] + "key" => ":streamer", + "value" => [%{"tuple" => [":workers", 5]}] } ], "need_reboot" => true @@ -454,7 +454,7 @@ test "saving config which need pleroma reboot", %{conn: conn} do end test "update setting which need reboot, don't change reboot flag until reboot", %{conn: conn} do - clear_config([:shout, :enabled], true) + clear_config([:streamer, :workers], 3) assert conn |> put_req_header("content-type", "application/json") @@ -462,17 +462,17 @@ test "update setting which need reboot, don't change reboot flag until reboot", "/api/pleroma/admin/config", %{ configs: [ - %{group: ":pleroma", key: ":shout", value: [%{"tuple" => [":enabled", true]}]} + %{group: ":pleroma", key: ":streamer", value: [%{"tuple" => [":workers", 5]}]} ] } ) |> json_response_and_validate_schema(200) == %{ "configs" => [ %{ - "db" => [":enabled"], + "db" => [":workers"], "group" => ":pleroma", - "key" => ":shout", - "value" => [%{"tuple" => [":enabled", true]}] + "key" => ":streamer", + "value" => [%{"tuple" => [":workers", 5]}] } ], "need_reboot" => true diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index f1f3a8b09a..773fd4be5b 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -1823,7 +1823,6 @@ test "verify_credentials" do response = json_response_and_validate_schema(conn, 200) assert %{"id" => id, "source" => %{"privacy" => "public"}} = response - assert response["pleroma"]["chat_token"] assert response["pleroma"]["unread_notifications_count"] == 6 assert id == to_string(user.id) end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 953e6008df..767c5e0b50 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -43,7 +43,6 @@ test "get instance information", %{conn: conn} do "background_upload_limit" => _, "banner_upload_limit" => _, "background_image" => from_config_background, - "shout_limit" => _, "description_limit" => _, "rules" => _, "pleroma" => %{ diff --git a/test/pleroma/web/shout_channel_test.exs b/test/pleroma/web/shout_channel_test.exs deleted file mode 100644 index e1de805934..0000000000 --- a/test/pleroma/web/shout_channel_test.exs +++ /dev/null @@ -1,41 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ShoutChannelTest do - use Pleroma.Web.ChannelCase - alias Pleroma.Web.ShoutChannel - alias Pleroma.Web.UserSocket - - import Pleroma.Factory - - setup do - user = insert(:user) - - {:ok, _, socket} = - socket(UserSocket, "", %{user_name: user.nickname}) - |> subscribe_and_join(ShoutChannel, "chat:public") - - {:ok, socket: socket} - end - - test "it broadcasts a message", %{socket: socket} do - push(socket, "new_msg", %{"text" => "why is tenshi eating a corndog so cute?"}) - assert_broadcast("new_msg", %{text: "why is tenshi eating a corndog so cute?"}) - end - - describe "message lengths" do - setup do: clear_config([:shout, :limit]) - - test "it ignores messages of length zero", %{socket: socket} do - push(socket, "new_msg", %{"text" => ""}) - refute_broadcast("new_msg", %{text: ""}) - end - - test "it ignores messages above a certain length", %{socket: socket} do - clear_config([:shout, :limit], 2) - push(socket, "new_msg", %{"text" => "123"}) - refute_broadcast("new_msg", %{text: "123"}) - end - end -end From 4442fc40e61882090f44b3954daabac54a7b81c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 3 Nov 2022 22:43:20 +0100 Subject: [PATCH 14/35] Add test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/language_detector.ex | 4 +-- .../language/language_detector_test.ex | 31 +++++++++++++++++++ test/pleroma/language/translation_test.ex | 5 ++- test/support/language_detector_mock.ex | 18 +++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 test/pleroma/language/language_detector_test.ex create mode 100644 test/support/language_detector_mock.ex diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index 3901a8b90b..b19eb45711 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -11,7 +11,7 @@ def missing_dependencies do if provider do provider.missing_dependencies() else - nil + [] end end @@ -28,7 +28,7 @@ def detect(text) do end end - defp get_provider() do + defp get_provider do Pleroma.Config.get([__MODULE__, :provider]) end end diff --git a/test/pleroma/language/language_detector_test.ex b/test/pleroma/language/language_detector_test.ex new file mode 100644 index 0000000000..4d9af33bf7 --- /dev/null +++ b/test/pleroma/language/language_detector_test.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.LanguageDetectorTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Language.LanguageDetector + + setup do: clear_config([Pleroma.Language.LanguageDetector, :provider], LanguageDetectorMock) + + test "it detects text language" do + detected_language = LanguageDetector.detect("Je viens d'atterrir en Tchéquie.") + + assert detected_language == "fr" + end + + test "it returns nil if text is not long enough" do + detected_language = LanguageDetector.detect("it returns nil") + + assert detected_language == nil + end + + test "it returns nil if no provider specified" do + clear_config([Pleroma.Language.LanguageDetector, :provider], nil) + + detected_language = LanguageDetector.detect("this should also return nil") + + assert detected_language == nil + end +end diff --git a/test/pleroma/language/translation_test.ex b/test/pleroma/language/translation_test.ex index ecab3d20f7..c518e74918 100644 --- a/test/pleroma/language/translation_test.ex +++ b/test/pleroma/language/translation_test.ex @@ -1,8 +1,11 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Language.TranslationTest do use Pleroma.Web.ConnCase alias Pleroma.Language.Translation - # use Oban.Testing, repo: Pleroma.Repo setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) diff --git a/test/support/language_detector_mock.ex b/test/support/language_detector_mock.ex new file mode 100644 index 0000000000..2a85dcd633 --- /dev/null +++ b/test/support/language_detector_mock.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule LanguageDetectorMock do + alias Pleroma.Language.LanguageDetector.Provider + + @behaviour Provider + + @impl Provider + def missing_dependencies, do: [] + + @impl Provider + def configured?, do: true + + @impl Provider + def detect(text), do: "fr" +end From 85105e0049c3a24e7204249f3c7e17d5f16a95a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 3 Nov 2022 23:31:36 +0100 Subject: [PATCH 15/35] Detect language for incoming posts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/activity_pub/transmogrifier.ex | 13 +++++++++++-- test/support/language_detector_mock.ex | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 2a0605fb9e..1de8107baf 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.Activity alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Language.LanguageDetector alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment @@ -23,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Workers.TransmogrifierWorker import Ecto.Query + import Pleroma.Web.CommonAPI.Utils, only: [get_valid_language: 1] import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] require Logger @@ -1100,8 +1102,9 @@ defp maybe_add_content_map(object), do: object def maybe_add_language(object) do language = - get_language_from_context(object) |> Pleroma.Web.CommonAPI.Utils.get_valid_language() || - get_language_from_content_map(object) |> Pleroma.Web.CommonAPI.Utils.get_valid_language() + get_language_from_context(object) |> get_valid_language() || + get_language_from_content_map(object) |> get_valid_language() || + get_language_from_content(object) |> get_valid_language() if language do Map.put(object, "language", language) @@ -1133,4 +1136,10 @@ defp get_language_from_content_map(%{"contentMap" => content_map, "content" => s end defp get_language_from_content_map(_), do: nil + + defp get_language_from_content(%{"summary" => summary, "content" => content}) do + LanguageDetector.detect("#{summary} #{content}") + end + + defp get_language_from_content(_), do: nil end diff --git a/test/support/language_detector_mock.ex b/test/support/language_detector_mock.ex index 2a85dcd633..3e6a258ae0 100644 --- a/test/support/language_detector_mock.ex +++ b/test/support/language_detector_mock.ex @@ -14,5 +14,5 @@ def missing_dependencies, do: [] def configured?, do: true @impl Provider - def detect(text), do: "fr" + def detect(_text), do: "fr" end From ade579633724ebd605a4ea27148d1b46ac9ab082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 4 Nov 2022 20:52:25 +0100 Subject: [PATCH 16/35] Improve getting language from context, add more tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/activity_pub/transmogrifier.ex | 15 +++- .../tesla_mock/deepl-translation.json | 1 + test/pleroma/language/translation/deepl.ex | 27 +++++++ .../transmogrifier/note_handling_test.exs | 81 +++++++++++++++++++ test/support/http_request_mock.ex | 9 +++ 5 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/tesla_mock/deepl-translation.json create mode 100644 test/pleroma/language/translation/deepl.ex diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 1de8107baf..d2fb21bd8a 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -532,6 +532,7 @@ def handle_incoming( |> fix_type(fetch_options) |> fix_in_reply_to(fetch_options) |> fix_quote_url(fetch_options) + |> maybe_add_language_from_activity(data) data = Map.put(data, "object", object) options = Keyword.put(options, :local, false) @@ -1113,6 +1114,16 @@ def maybe_add_language(object) do end end + def maybe_add_language_from_activity(object, activity) do + language = get_language_from_context(activity) |> get_valid_language() + + if language do + Map.put(object, "language", language) + else + object + end + end + defp get_language_from_context(%{"@context" => context}) when is_list(context) do case context |> Enum.find(fn @@ -1137,8 +1148,8 @@ defp get_language_from_content_map(%{"contentMap" => content_map, "content" => s defp get_language_from_content_map(_), do: nil - defp get_language_from_content(%{"summary" => summary, "content" => content}) do - LanguageDetector.detect("#{summary} #{content}") + defp get_language_from_content(%{"content" => content}) do + LanguageDetector.detect(content) end defp get_language_from_content(_), do: nil diff --git a/test/fixtures/tesla_mock/deepl-translation.json b/test/fixtures/tesla_mock/deepl-translation.json new file mode 100644 index 0000000000..fef7bb2150 --- /dev/null +++ b/test/fixtures/tesla_mock/deepl-translation.json @@ -0,0 +1 @@ +{"translations":[{"detected_source_language":"PL","text":"REMOVE THE FOLLOWER!Paste this on your follower. If we get 70% of nk users...they will remove the follower!!!"}]} \ No newline at end of file diff --git a/test/pleroma/language/translation/deepl.ex b/test/pleroma/language/translation/deepl.ex new file mode 100644 index 0000000000..3d490a0819 --- /dev/null +++ b/test/pleroma/language/translation/deepl.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.TranslationTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Language.Translation.Deepl + + test "it translates text" do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + clear_config([Pleroma.Language.Translation.Deepl, :plan], :free) + clear_config([Pleroma.Language.Translation.Deepl, :api_key], "API_KEY") + + {:ok, res} = + Deepl.translate( + "USUNĄĆ ŚLEDZIKA!Wklej to na swojego śledzika. Jeżeli uzbieramy 70% użytkowników nk...to usuną śledzika!!!", + "pl", + "en" + ) + + assert %{ + detected_source_language: "PL", + provider: "DeepL" + } = res + end +end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index c9d46ca2c6..95e43f5545 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -385,6 +385,87 @@ test "it correctly processes messages with weirdness in address fields" do assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"] assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] end + + test "it detects language from context" do + user = insert(:user) + + message = %{ + "@context" => ["https://www.w3.org/ns/activitystreams", %{"@language" => "pl"}], + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Create", + "object" => %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "id" => Utils.generate_object_id(), + "type" => "Note", + "content" => "Szczęść Boże", + "attributedTo" => user.ap_id + }, + "actor" => user.ap_id + } + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message) + object = Object.normalize(data["object"], fetch: false) + + assert object.data["language"] == "pl" + end + + test "it detects language from contentMap" do + user = insert(:user) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Create", + "object" => %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "id" => Utils.generate_object_id(), + "type" => "Note", + "content" => "Szczęść Boże", + "contentMap" => %{ + "de" => "Gott segne", + "pl" => "Szczęść Boże" + }, + "attributedTo" => user.ap_id + }, + "actor" => user.ap_id + } + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message) + object = Object.normalize(data["object"], fetch: false) + + assert object.data["language"] == "pl" + end + + test "it detects language from content" do + clear_config([Pleroma.Language.LanguageDetector, :provider], LanguageDetectorMock) + + user = insert(:user) + + message = %{ + "@context" => ["https://www.w3.org/ns/activitystreams"], + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Create", + "object" => %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "id" => Utils.generate_object_id(), + "type" => "Note", + "content" => "Dieu vous bénisse, Fédivers.", + "attributedTo" => user.ap_id + }, + "actor" => user.ap_id + } + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message) + object = Object.normalize(data["object"], fetch: false) + + assert object.data["language"] == "fr" + end end describe "`handle_incoming/2`, Mastodon format `replies` handling" do diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index eab8639aba..d39a6f7b4e 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1560,6 +1560,15 @@ def post("http://404.site" <> _, _, _, _) do }} end + def post("https://api-free.deepl.com/v2/translate" <> _, _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/deepl-translation.json"), + headers: [{"content-type", "application/json"}] + }} + end + def post(url, query, body, headers) do {:error, "Mock response not implemented for POST #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} From 6649eb84a2b14c18574e9fa6d6a8b5f38b7b805c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 4 Nov 2022 21:22:29 +0100 Subject: [PATCH 17/35] Add all 639-1 languages to @supported_locales MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/translation/deepl.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 10 +++++----- .../language/translation/{deepl.ex => deepl_test.ex} | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) rename test/pleroma/language/translation/{deepl.ex => deepl_test.ex} (70%) diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 81048378c3..6942f0477d 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Language.Translation.Deepl do @impl Provider def configured? do - not_empty_string(get_plan()) and not_empty_string(get_api_key()) + is_atom(get_plan()) and not_empty_string(get_api_key()) end @impl Provider diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1af1b19839..8de329e19e 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -25,11 +25,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do @supported_locales ~w( aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce ch co cr cs cu cv cy da - de dv dz ee el en eo es et eu fa ff fi fj fo fr fy ga gd gl gu gv ha he hi ho hr ht hu hy hz ia - id ie ig ii ik io is it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln lo - lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv ny oc oj om or os pa pi pl - ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl sn so sq sr ss st su sv sw ta te tg th ti tk tl - tn to tr ts tt tw ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu ast ckb kab kmr zgh + de dv dz ee el en eo es et eu fa ff fi fj fo fr fy ga gd gl gn gu gv ha he hi ho hr ht hu hy hz + ia id ie ig ii ik io is it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln + lo lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv ny oc oj om or os pa pi + pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl sm sn so sq sr ss st su sv sw ta te tg th ti + tk tl tn to tr ts tt tw ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu ast ckb kab kmr zgh ) def attachments_from_ids(%{media_ids: ids, descriptions: desc}) do diff --git a/test/pleroma/language/translation/deepl.ex b/test/pleroma/language/translation/deepl_test.ex similarity index 70% rename from test/pleroma/language/translation/deepl.ex rename to test/pleroma/language/translation/deepl_test.ex index 3d490a0819..9acda15dab 100644 --- a/test/pleroma/language/translation/deepl.ex +++ b/test/pleroma/language/translation/deepl_test.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Language.TranslationTest do +defmodule Pleroma.Language.Translation.DeeplTest do use Pleroma.Web.ConnCase alias Pleroma.Language.Translation.Deepl @@ -13,11 +13,11 @@ test "it translates text" do clear_config([Pleroma.Language.Translation.Deepl, :api_key], "API_KEY") {:ok, res} = - Deepl.translate( - "USUNĄĆ ŚLEDZIKA!Wklej to na swojego śledzika. Jeżeli uzbieramy 70% użytkowników nk...to usuną śledzika!!!", - "pl", - "en" - ) + Deepl.translate( + "USUNĄĆ ŚLEDZIKA!Wklej to na swojego śledzika. Jeżeli uzbieramy 70% użytkowników nk...to usuną śledzika!!!", + "pl", + "en" + ) assert %{ detected_source_language: "PL", From 0c1eaf26341b1615689149f46c424d15fb098c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Nov 2022 13:05:14 +0100 Subject: [PATCH 18/35] Add contact account to InstanceView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 6 ++++++ .../web/mastodon_api/views/instance_view.ex | 21 ++++++++++++++++++- .../controllers/instance_controller_test.exs | 14 +++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/config/description.exs b/config/description.exs index bdf57dd633..b038051063 100644 --- a/config/description.exs +++ b/config/description.exs @@ -566,6 +566,12 @@ "Cool instance" ] }, + %{ + key: :contact_username, + type: :string, + description: "Instance owner username", + suggestions: ["admin"] + }, %{ key: :limit, type: :integer, diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f8852e9a45..7fe150ccb4 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -6,7 +6,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do use Pleroma.Web, :view alias Pleroma.Config + alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF + alias Pleroma.Web.MastodonAPI @mastodon_api_level "2.7.2" @@ -31,6 +33,7 @@ def render("show.json", _) do registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), configuration: configuration(), + contact_account: contact_account(Keyword.get(instance, :contact_username)), rules: render(__MODULE__, "rules.json"), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), @@ -74,7 +77,7 @@ def render("show2.json", _) do }, contact: %{ email: Keyword.get(instance, :email), - account: nil + account: contact_account(Keyword.get(instance, :contact_username)) }, rules: render(__MODULE__, "rules.json"), # Extra (not present in Mastodon): @@ -250,4 +253,20 @@ defp pleroma_configuration2(instance) do }) }) end + + defp contact_account(nil), do: nil + + defp contact_account("@" <> username) do + contact_account(username) + end + + defp contact_account(username) do + user = User.get_cached_by_nickname(username) + + if user do + MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) + else + nil + end + end end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 953e6008df..4db9b4da8f 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -139,9 +139,19 @@ test "get oauth_consumer_strategies", %{conn: conn} do assert result["pleroma"]["oauth_consumer_strategies"] == ["keycloak"] end - test "get instance information v2", %{conn: conn} do - clear_config([:auth, :oauth_consumer_strategies], []) + test "get instance contact information", %{conn: conn} do + user = insert(:user, %{local: true}) + clear_config([:instance, :contact_username], user.nickname) + + conn = get(conn, "/api/v1/instance") + + assert result = json_response_and_validate_schema(conn, 200) + + assert result["contact_account"]["id"] == user.id + end + + test "get instance information v2", %{conn: conn} do assert get(conn, "/api/v2/instance") |> json_response_and_validate_schema(200) end From 87e9bbf86c213235d4964c5a1e1ec04814e72737 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 5 Nov 2022 13:56:56 -0500 Subject: [PATCH 19/35] Add RemoteReportPolicy to reject reports without enough information --- config/config.exs | 4 + .../activity_pub/mrf/remote_report_policy.ex | 80 +++++++++++++++++ .../mrf/remote_report_policy_test.exs | 85 +++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex create mode 100644 test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs diff --git a/config/config.exs b/config/config.exs index 7a621cbc2f..fa50f83683 100644 --- a/config/config.exs +++ b/config/config.exs @@ -428,6 +428,10 @@ config :pleroma, :mrf_inline_quote, prefix: "RT" +config :pleroma, :mrf_remote_report, + reject_anonymous: true, + reject_empty_message: true + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex new file mode 100644 index 0000000000..3cf47e3ed8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -0,0 +1,80 @@ +defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do + @moduledoc "Drop remote reports if they don't contain enough information." + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Config + + @impl true + def filter(%{"type" => "Flag"} = object) do + with {_, false} <- {:local, local?(object)}, + {:ok, _} <- maybe_reject_anonymous(object), + {:ok, _} <- maybe_reject_empty_message(object) do + {:ok, object} + else + {:local, true} -> {:ok, object} + {:reject, message} -> {:reject, message} + error -> {:reject, error} + end + end + + def filter(object), do: {:ok, object} + + defp maybe_reject_anonymous(%{"actor" => actor} = object) do + with true <- Config.get([:mrf_remote_report, :reject_anonymous]), + %URI{path: "/actor"} <- URI.parse(actor) do + {:reject, "[RemoteReportPolicy] Anonymous: #{actor}"} + else + _ -> {:ok, object} + end + end + + defp maybe_reject_empty_message(%{"content" => content} = object) + when is_binary(content) and content != "" do + {:ok, object} + end + + defp maybe_reject_empty_message(object) do + if Config.get([:mrf_remote_report, :reject_empty_message]) do + {:reject, ["RemoteReportPolicy] No content"]} + else + {:ok, object} + end + end + + defp local?(%{"actor" => actor}) do + String.starts_with?(actor, Pleroma.Web.Endpoint.url()) + end + + @impl true + def describe do + mrf_remote_report = + Config.get(:mrf_remote_report) + |> Enum.into(%{}) + + {:ok, %{mrf_remote_report: mrf_remote_report}} + end + + @impl true + def config_description do + %{ + key: :mrf_remote_report, + related_policy: "Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy", + label: "MRF Remote Report", + description: "Drop remote reports if they don't contain enough information.", + children: [ + %{ + key: :reject_anonymous, + type: :boolean, + description: "Reject anonymous remote reports?", + suggestions: [true] + }, + %{ + key: :reject_empty_message, + type: :boolean, + description: "Reject remote reports with no message?", + suggestions: [true] + } + ] + } + end +end diff --git a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs new file mode 100644 index 0000000000..55fa0f2f20 --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs @@ -0,0 +1,85 @@ +defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicyTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy + + test "doesn't impact local report" do + clear_config([:mrf_remote_report, :reject_anonymous], true) + clear_config([:mrf_remote_report, :reject_empty_message], true) + + activity = %{ + "type" => "Flag", + "actor" => "http://localhost:4001/actor" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + + test "rejects anonymous report if `reject_anonymous: true`" do + clear_config([:mrf_remote_report, :reject_anonymous], true) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/actor" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end + + test "preserves anonymous report if `reject_anonymous: false`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/actor" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + + test "rejects empty message report if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_empty_message], true) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end + + test "rejects empty message report (\"\") if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_empty_message], true) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron", + "content" => "" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end + + test "preserves empty message report if `reject_empty_message: false`" do + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + + test "preserves anonymous, empty message report with all settings disabled" do + clear_config([:mrf_remote_report, :reject_empty_message], false) + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/actor" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end +end From 3957167a9dced332160447f451eba168f0bdcb95 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 5 Nov 2022 14:07:37 -0500 Subject: [PATCH 20/35] RemoteReportPolicy: add `:reject_all` option, fix tests --- config/config.exs | 1 + .../activity_pub/mrf/remote_report_policy.ex | 15 +++++++++++ .../mrf/remote_report_policy_test.exs | 25 ++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index fa50f83683..364ade161f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -429,6 +429,7 @@ config :pleroma, :mrf_inline_quote, prefix: "RT" config :pleroma, :mrf_remote_report, + reject_all: false, reject_anonymous: true, reject_empty_message: true diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index 3cf47e3ed8..0bd83d8f00 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do @impl true def filter(%{"type" => "Flag"} = object) do with {_, false} <- {:local, local?(object)}, + {:ok, _} <- maybe_reject_all(object), {:ok, _} <- maybe_reject_anonymous(object), {:ok, _} <- maybe_reject_empty_message(object) do {:ok, object} @@ -19,6 +20,14 @@ def filter(%{"type" => "Flag"} = object) do def filter(object), do: {:ok, object} + defp maybe_reject_all(object) do + if Config.get([:mrf_remote_report, :reject_all]) do + {:reject, "[RemoteReportPolicy] Remote report"} + else + {:ok, object} + end + end + defp maybe_reject_anonymous(%{"actor" => actor} = object) do with true <- Config.get([:mrf_remote_report, :reject_anonymous]), %URI{path: "/actor"} <- URI.parse(actor) do @@ -62,6 +71,12 @@ def config_description do label: "MRF Remote Report", description: "Drop remote reports if they don't contain enough information.", children: [ + %{ + key: :reject_all, + type: :boolean, + description: "Reject all remote reports? (this option takes precedence)", + suggestions: [false] + }, %{ key: :reject_anonymous, type: :boolean, diff --git a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs index 55fa0f2f20..43258a7f60 100644 --- a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs @@ -3,6 +3,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicyTest do alias Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy + setup do + clear_config([:mrf_remote_report, :reject_all], false) + end + test "doesn't impact local report" do clear_config([:mrf_remote_report, :reject_anonymous], true) clear_config([:mrf_remote_report, :reject_empty_message], true) @@ -17,6 +21,7 @@ test "doesn't impact local report" do test "rejects anonymous report if `reject_anonymous: true`" do clear_config([:mrf_remote_report, :reject_anonymous], true) + clear_config([:mrf_remote_report, :reject_empty_message], true) activity = %{ "type" => "Flag", @@ -28,6 +33,7 @@ test "rejects anonymous report if `reject_anonymous: true`" do test "preserves anonymous report if `reject_anonymous: false`" do clear_config([:mrf_remote_report, :reject_anonymous], false) + clear_config([:mrf_remote_report, :reject_empty_message], false) activity = %{ "type" => "Flag", @@ -38,6 +44,7 @@ test "preserves anonymous report if `reject_anonymous: false`" do end test "rejects empty message report if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], true) activity = %{ @@ -49,6 +56,7 @@ test "rejects empty message report if `reject_empty_message: true`" do end test "rejects empty message report (\"\") if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], true) activity = %{ @@ -61,6 +69,7 @@ test "rejects empty message report (\"\") if `reject_empty_message: true`" do end test "preserves empty message report if `reject_empty_message: false`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], false) activity = %{ @@ -72,7 +81,7 @@ test "preserves empty message report if `reject_empty_message: false`" do end test "preserves anonymous, empty message report with all settings disabled" do - clear_config([:mrf_remote_report, :reject_empty_message], false) + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], false) activity = %{ @@ -82,4 +91,18 @@ test "preserves anonymous, empty message report with all settings disabled" do assert {:ok, _} = RemoteReportPolicy.filter(activity) end + + test "reject remote report if `reject_all: true`" do + clear_config([:mrf_remote_report, :reject_all], true) + clear_config([:mrf_remote_report, :reject_anonymous], false) + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron", + "content" => "Transphobia" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end end From 19885d9347b3bf677836813d22b3c75c125862e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Nov 2022 20:16:32 +0100 Subject: [PATCH 21/35] Fix fasttext for multiline posts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/language/language_detector/fasttext.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/language/language_detector/fasttext.ex b/lib/pleroma/language/language_detector/fasttext.ex index d479d21255..0f621a000b 100644 --- a/lib/pleroma/language/language_detector/fasttext.ex +++ b/lib/pleroma/language/language_detector/fasttext.ex @@ -25,7 +25,7 @@ def configured?, do: not_empty_string(get_model()) def detect(text) do text_path = Path.join(System.tmp_dir!(), "fasttext-#{Ecto.UUID.generate()}") - File.write(text_path, text) + File.write(text_path, text |> String.replace(~r/\s+/, " ")) detected_language = case System.cmd("fasttext", ["predict", get_model(), text_path]) do From 4f042374d80564ae80c971617293118e45b81e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Nov 2022 21:41:58 +0100 Subject: [PATCH 22/35] Deepl: use :base_url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 10 +++++----- lib/pleroma/language/translation/deepl.ex | 15 ++++----------- test/pleroma/language/translation/deepl_test.ex | 2 +- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/config/description.exs b/config/description.exs index f3395863c8..f497a12ff3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3567,10 +3567,10 @@ }, %{ group: {:subgroup, Pleroma.Language.Translation.Deepl}, - key: :plan, - label: "DeepL plan", - type: {:dropdown, :atom}, - suggestions: [:free, :pro] + key: :base_url, + label: "DeepL base URL", + type: :string, + suggestions: ["https://api-free.deepl.com", "https://api.deepl.com"] }, %{ group: {:subgroup, Pleroma.Language.Translation.Deepl}, @@ -3582,7 +3582,7 @@ %{ group: {:subgroup, Pleroma.Language.Translation.Libretranslate}, key: :base_url, - label: "LibreTranslate plan", + label: "LibreTranslate instance URL", type: :string, suggestions: ["https://libretranslate.com"] }, diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 6942f0477d..5a34740906 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -11,12 +11,12 @@ defmodule Pleroma.Language.Translation.Deepl do @impl Provider def configured? do - is_atom(get_plan()) and not_empty_string(get_api_key()) + is_atom(get_base_url()) and not_empty_string(get_api_key()) end @impl Provider def translate(content, source_language, target_language) do - endpoint = endpoint_url() + endpoint = get_base_url() case Pleroma.HTTP.post( endpoint <> @@ -58,15 +58,8 @@ def translate(content, source_language, target_language) do end end - defp endpoint_url do - case get_plan() do - :free -> "https://api-free.deepl.com/v2/translate" - _ -> "https://api.deepl.com/v2/translate" - end - end - - defp get_plan do - Pleroma.Config.get([__MODULE__, :plan]) + defp get_base_url do + Pleroma.Config.get([__MODULE__, :base_url]) end defp get_api_key do diff --git a/test/pleroma/language/translation/deepl_test.ex b/test/pleroma/language/translation/deepl_test.ex index 9acda15dab..0c29b84a4e 100644 --- a/test/pleroma/language/translation/deepl_test.ex +++ b/test/pleroma/language/translation/deepl_test.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Language.Translation.DeeplTest do test "it translates text" do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) - clear_config([Pleroma.Language.Translation.Deepl, :plan], :free) + clear_config([Pleroma.Language.Translation.Deepl, :base_url], "https://api-free.deepl.com") clear_config([Pleroma.Language.Translation.Deepl, :api_key], "API_KEY") {:ok, res} = From dbfc460e37fb1218ea9207925fed48549f21f194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Nov 2022 21:54:50 +0100 Subject: [PATCH 23/35] Add fasttext configuration to Docker image(?) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- Dockerfile | 7 +++++-- config/docker.exs | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3a662796e9..cd42248cae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG MIX_ENV=prod \ WORKDIR /src RUN apt-get update &&\ - apt-get install -y git elixir erlang-dev erlang-nox build-essential cmake libssl-dev libmagic-dev automake autoconf libncurses5-dev &&\ + apt-get install -y git elixir erlang-dev erlang-nox build-essential cmake libssl-dev libmagic-dev automake autoconf libncurses5-dev fasttext &&\ mix local.hex --force &&\ mix local.rebar --force @@ -46,7 +46,10 @@ RUN apt-get update &&\ mkdir -p ${DATA}/static &&\ chown -R pleroma ${DATA} &&\ mkdir -p /etc/pleroma &&\ - chown -R pleroma /etc/pleroma + chown -R pleroma /etc/pleroma &&\ + mkdir -p /usr/share/fasttext &&\ + curl -L https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -o /usr/share/fasttext/lid.176.bin &&\ + chmod 0644 /usr/share/fasttext/lid.176.bin USER pleroma diff --git a/config/docker.exs b/config/docker.exs index 4d9c05e5b5..40ea225b62 100644 --- a/config/docker.exs +++ b/config/docker.exs @@ -32,6 +32,12 @@ config :pleroma, :instance, static_dir: "/var/lib/pleroma/static" config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads" +config :pleroma, Pleroma.Language.LanguageDetector, + provider: Pleroma.Language.LanguageDetector.Fasttext + +config :pleroma, Pleroma.Language.LanguageDetector.Fasttext, + model: "/usr/share/fasttext/lid.176.bin" + # We can't store the secrets in this file, since this is baked into the docker image if not File.exists?("/var/lib/pleroma/secret.exs") do secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64) From 61cf21fb205eee0949b0a4ffa6c80fc8c5fe772f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 01:58:49 +0000 Subject: [PATCH 24/35] Dockerfile: add curl --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cd42248cae..e46bc44fdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ ARG HOME=/opt/pleroma ARG DATA=/var/lib/pleroma RUN apt-get update &&\ - apt-get install -y --no-install-recommends imagemagick libmagic-dev ffmpeg libimage-exiftool-perl libncurses5 postgresql-client &&\ + apt-get install -y --no-install-recommends curl imagemagick libmagic-dev ffmpeg libimage-exiftool-perl libncurses5 postgresql-client &&\ adduser --system --shell /bin/false --home ${HOME} pleroma &&\ mkdir -p ${DATA}/uploads &&\ mkdir -p ${DATA}/static &&\ From f884273bb9f966e9082d918e35b3a34bd77d2013 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 02:00:24 +0000 Subject: [PATCH 25/35] Dockerfile: move fasttext to runtime image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e46bc44fdb..985414d142 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG MIX_ENV=prod \ WORKDIR /src RUN apt-get update &&\ - apt-get install -y git elixir erlang-dev erlang-nox build-essential cmake libssl-dev libmagic-dev automake autoconf libncurses5-dev fasttext &&\ + apt-get install -y git elixir erlang-dev erlang-nox build-essential cmake libssl-dev libmagic-dev automake autoconf libncurses5-dev &&\ mix local.hex --force &&\ mix local.rebar --force @@ -40,7 +40,7 @@ ARG HOME=/opt/pleroma ARG DATA=/var/lib/pleroma RUN apt-get update &&\ - apt-get install -y --no-install-recommends curl imagemagick libmagic-dev ffmpeg libimage-exiftool-perl libncurses5 postgresql-client &&\ + apt-get install -y --no-install-recommends curl imagemagick libmagic-dev ffmpeg libimage-exiftool-perl libncurses5 postgresql-client fasttext &&\ adduser --system --shell /bin/false --home ${HOME} pleroma &&\ mkdir -p ${DATA}/uploads &&\ mkdir -p ${DATA}/static &&\ From 9876794c8a979c724d78cbe3e172d9bdaccca097 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 14:14:23 +0000 Subject: [PATCH 26/35] Docker: try adding ca-certificates --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 985414d142..530c6f64f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ ARG HOME=/opt/pleroma ARG DATA=/var/lib/pleroma RUN apt-get update &&\ - apt-get install -y --no-install-recommends curl imagemagick libmagic-dev ffmpeg libimage-exiftool-perl libncurses5 postgresql-client fasttext &&\ + apt-get install -y --no-install-recommends curl ca-certificates imagemagick libmagic-dev ffmpeg libimage-exiftool-perl libncurses5 postgresql-client fasttext &&\ adduser --system --shell /bin/false --home ${HOME} pleroma &&\ mkdir -p ${DATA}/uploads &&\ mkdir -p ${DATA}/static &&\ From fc81e325e75b8b657dd62367df1861365acf18b5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 14:16:56 +0000 Subject: [PATCH 27/35] DeepL: not_empty_string --- lib/pleroma/language/translation/deepl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 5a34740906..7771c8fe17 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Language.Translation.Deepl do @impl Provider def configured? do - is_atom(get_base_url()) and not_empty_string(get_api_key()) + not_empty_string(get_base_url()) and not_empty_string(get_api_key()) end @impl Provider From 5e6bb2aed018f7a5392f05156de63d8685ef1de2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 08:24:54 -0600 Subject: [PATCH 28/35] DeepL: fix endpoint --- lib/pleroma/language/translation/deepl.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 7771c8fe17..f43abb1c0d 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -16,7 +16,7 @@ def configured? do @impl Provider def translate(content, source_language, target_language) do - endpoint = get_base_url() + endpoint = get_endpoint() case Pleroma.HTTP.post( endpoint <> @@ -58,6 +58,10 @@ def translate(content, source_language, target_language) do end end + defp get_endpoint do + URI.merge(get_base_url(), "/v2/translate") + end + defp get_base_url do Pleroma.Config.get([__MODULE__, :base_url]) end From ee58bcafc9b6f5306a85d354d76ec600fc9c68f4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 08:29:03 -0600 Subject: [PATCH 29/35] DeepL: endpoint to_string() --- lib/pleroma/language/translation/deepl.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index f43abb1c0d..325ef0861a 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -59,7 +59,9 @@ def translate(content, source_language, target_language) do end defp get_endpoint do - URI.merge(get_base_url(), "/v2/translate") + get_base_url() + |> URI.merge("/v2/translate") + |> URI.to_string() end defp get_base_url do From 22edf882a58d698ac26093d0b515cf7493e9925a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 12:02:42 -0600 Subject: [PATCH 30/35] MediaProxy: use SHA-256 --- lib/pleroma/web/media_proxy.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy.ex b/lib/pleroma/web/media_proxy.ex index d64760fc26..eab37488a8 100644 --- a/lib/pleroma/web/media_proxy.ex +++ b/lib/pleroma/web/media_proxy.ex @@ -127,7 +127,7 @@ def decode_url(encoded) do end defp signed_url(url) do - :crypto.mac(:hmac, :sha, Config.get([Endpoint, :secret_key_base]), url) + :crypto.mac(:hmac, :sha256, Config.get([Endpoint, :secret_key_base]), url) end def filename(url_or_path) do From 272e022a71031d4b12f345b3749af9e0493df3b3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Nov 2022 13:27:45 -0600 Subject: [PATCH 31/35] OpenGraphTest: fix MediaProxy URL --- test/pleroma/web/metadata/providers/open_graph_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/metadata/providers/open_graph_test.exs b/test/pleroma/web/metadata/providers/open_graph_test.exs index b7ce95f7db..fde8d12f22 100644 --- a/test/pleroma/web/metadata/providers/open_graph_test.exs +++ b/test/pleroma/web/metadata/providers/open_graph_test.exs @@ -144,7 +144,7 @@ test "video attachments have image thumbnail with WxH metadata with Preview Prox [ property: "og:image", content: - "http://localhost:4001/proxy/preview/LzAnlke-l5oZbNzWsrHfprX1rGw/aHR0cHM6Ly9wbGVyb21hLmdvdi9hYm91dC9qdWNoZS53ZWJt/juche.webm" + "http://localhost:4001/proxy/preview/YyXfYXyaEAGVHVfKF-HRc4GrFMY03Smi7effZ8WKow8/aHR0cHM6Ly9wbGVyb21hLmdvdi9hYm91dC9qdWNoZS53ZWJt/juche.webm" ], []} in result end From 728643b623924144c71143a5780f6039e0ce42f6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:06:16 -0600 Subject: [PATCH 32/35] LanguageDetector: strip non-language text to (hopefully) improve accuracy --- lib/pleroma/language/language_detector.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index b19eb45711..0be69d220c 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -15,10 +15,18 @@ def missing_dependencies do end end + # Strip tags from text, etc. + defp prepare_text(text) do + text + |> Floki.parse_fragment!() + |> Floki.filter_out(".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre") + |> Floki.text() + end + def detect(text) do provider = get_provider() - {:ok, text} = text |> FastSanitize.strip_tags() + text = prepare_text(text) word_count = text |> String.split(~r/\s+/) |> Enum.count() if word_count < @words_threshold or !provider or !provider.configured? do From f844c4ba13231a76ac163c773ac4ffeb8268c81a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:23:41 -0600 Subject: [PATCH 33/35] ActivityDraft: detect language from content_html so it can strip links --- lib/pleroma/web/common_api/activity_draft.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index fdc9999f55..dc7b4c6097 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -236,7 +236,7 @@ defp sensitive(draft) do defp language(draft) do language = Utils.get_valid_language(draft.params[:language]) || - LanguageDetector.detect(draft.full_payload) + LanguageDetector.detect(draft.content_html <> " " <> draft.summary) %__MODULE__{draft | language: language} end From 0ed5f9dcab41bd327668df51d8d3dab1992ceff5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:25:09 -0600 Subject: [PATCH 34/35] Docker: use the lightweight fasttext model --- Dockerfile | 6 +++--- config/docker.exs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 530c6f64f1..fd521245bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ LABEL maintainer="hello@soapbox.pub" \ org.opencontainers.image.description="Rebased" \ org.opencontainers.image.authors="hello@soapbox.pub" \ org.opencontainers.image.vendor="soapbox.pub" \ - org.opencontainers.image.documentation="https://gitlab.com/soapbox-pub/soapbox-be" \ + org.opencontainers.image.documentation="https://gitlab.com/soapbox-pub/rebased" \ org.opencontainers.image.licenses="AGPL-3.0" \ org.opencontainers.image.url="https://soapbox.pub" \ org.opencontainers.image.revision=$VCS_REF \ @@ -48,8 +48,8 @@ RUN apt-get update &&\ mkdir -p /etc/pleroma &&\ chown -R pleroma /etc/pleroma &&\ mkdir -p /usr/share/fasttext &&\ - curl -L https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -o /usr/share/fasttext/lid.176.bin &&\ - chmod 0644 /usr/share/fasttext/lid.176.bin + curl -L https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz -o /usr/share/fasttext/lid.176.ftz &&\ + chmod 0644 /usr/share/fasttext/lid.176.ftz USER pleroma diff --git a/config/docker.exs b/config/docker.exs index 40ea225b62..b204d89edf 100644 --- a/config/docker.exs +++ b/config/docker.exs @@ -36,7 +36,7 @@ provider: Pleroma.Language.LanguageDetector.Fasttext config :pleroma, Pleroma.Language.LanguageDetector.Fasttext, - model: "/usr/share/fasttext/lid.176.bin" + model: "/usr/share/fasttext/lid.176.ftz" # We can't store the secrets in this file, since this is baked into the docker image if not File.exists?("/var/lib/pleroma/secret.exs") do From 6dc55531db10690f926f63ab9b5f83623d56ef07 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:47:17 -0600 Subject: [PATCH 35/35] mix format --- lib/pleroma/language/language_detector.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index 0be69d220c..42d200a287 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -19,7 +19,9 @@ def missing_dependencies do defp prepare_text(text) do text |> Floki.parse_fragment!() - |> Floki.filter_out(".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre") + |> Floki.filter_out( + ".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre" + ) |> Floki.text() end