bigbuffet/pleroma
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

MultiLanguage: Remove map_to_str

Browse source 
Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
This commit is contained in:
marcin mikołajczak 2024-05-31 23:07:28 +02:00
parent 9b0c05f70c
commit 48be78b375
26 changed files with 115 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config/config.exs
View file

@ -940,12 +940,6 @@
config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fetch: true
config :pleroma, Pleroma.MultiLanguage,
template: "<div lang=\"{code}\">{content}</div>",
separator: "<br><hr><br>",
single_line_template: "[{code}] {content}",
single_line_separator: " | "
config :pleroma, Pleroma.Language.Translation, allow_unauthenticated: false, allow_remote: true
config :geospatial, Geospatial.Service, service: Geospatial.Providers.Nominatim

39
lib/pleroma/multi_language.ex
View file

@ -3,12 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.MultiLanguage do
defp template(:multi), do: Pleroma.Config.get([__MODULE__, :template])
defp template(:single), do: Pleroma.Config.get([__MODULE__, :single_line_template])
defp sep(:multi), do: Pleroma.Config.get([__MODULE__, :separator])
defp sep(:single), do: Pleroma.Config.get([__MODULE__, :single_line_separator])
def good_locale_code?(code) when is_binary(code), do: code =~ ~r<^[a-zA-Z0-9\-]+$>
def good_locale_code?(_code), do: false
@ -37,27 +31,6 @@ def validate_map(%{} = object) do
def validate_map(_), do: {:error, nil}
def map_to_str(data, opts \\ []) do
map_to_str_impl(data, if(opts[:multiline], do: :multi, else: :single))
end
defp map_to_str_impl(data, mode) do
with ks <- Map.keys(data),
[_, _ | _] <- ks,
ks <- Enum.sort(ks) do
template = template(mode)
ks
|> Enum.map(fn lang ->
format_template(template, %{code: lang, content: data[lang]})
end)
|> Enum.join(sep(mode))
else
[lang] -> data[lang]
_ -> nil
end
end
def str_to_map(data, opts \\ []) do
with lang when is_binary(lang) <- opts[:lang],
true <- good_locale_code?(lang) do
@ -67,16 +40,4 @@ def str_to_map(data, opts \\ []) do
%{"und" => data}
end
end
def format_template(template, %{code: code, content: content}) do
template
|> String.replace(
["{code}", "{content}"],
fn
"{code}" -> code
"{content}" -> content
end,
global: true
)
end
end

20
lib/pleroma/upload.ex
View file

@ -91,23 +91,23 @@ defp get_description(upload) do
end
defp validate_description_limit(%{} = description) do
len = Enum.reduce(description, 0, fn {_, content}, acc -> String.length(content) + acc end)
len <= Pleroma.Config.get([:instance, :description_limit])
Enum.each(description, fn content ->
String.length(content) <= Pleroma.Config.get([:instance, :description_limit])
end)
end
defp validate_description_limit(description) when is_binary(description) do
String.length(description) <= Pleroma.Config.get([:instance, :description_limit])
end
defp description_fields(%{} = description) do
defp description_fields(%{} = description, language) do
%{
"name" => Pleroma.MultiLanguage.map_to_str(description, multiline: false),
"name" => Map.get(description, language),
"nameMap" => description
}
end
defp description_fields(description) when is_binary(description) do
defp description_fields(description, _) when is_binary(description) do
%{"name" => description}
end
@ -121,6 +121,9 @@ def store(upload, opts \\ []) do
{:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
description = get_description(upload),
{_, true} <- {:description_limit, validate_description_limit(description)},
{_, true} <-
{:valid_locale,
opts[:language] == nil or Pleroma.MultiLanguage.good_locale_code?(opts[:language])},
{:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do
{:ok,
%{
@ -137,8 +140,9 @@ def store(upload, opts \\ []) do
|> Maps.put_if_present("height", upload.height)
]
}
|> Map.merge(description_fields(description))
|> Maps.put_if_present("blurhash", upload.blurhash)}
|> Map.merge(description_fields(description, opts[:language]))
|> Maps.put_if_present("blurhash", upload.blurhash)
|> Maps.put_if_present("language", opts[:language])}
else
{:description_limit, _} ->
{:error, :description_too_long}

24
lib/pleroma/web/activity_pub/activity_pub.ex
View file

@ -1593,27 +1593,37 @@ def fetch_activities_bounded(
|> Enum.reverse()
end
defp validate_media_description_map(%{} = map) do
with {:ok, %{}} <- Pleroma.MultiLanguage.validate_map(map) do
defp validate_media_description_map(%{} = map, language) do
with {:ok, %{}} <- Pleroma.MultiLanguage.validate_map(map),
true <- Pleroma.MultiLanguage.good_locale_code?(language) do
:ok
else
false -> :invalid_language
_ -> :error
end
end
defp validate_media_description_map(nil), do: :ok
defp validate_media_description_map(_), do: :error
defp validate_media_description_map(nil, _), do: :ok
defp validate_media_description_map(_, _), do: :error
@spec upload(Upload.source(), keyword()) :: {:ok, Object.t()} | {:error, any()}
def upload(file, opts \\ []) do
with {_, :ok} <- {:description_map, validate_media_description_map(opts[:description_map])},
with {_, :ok} <-
{:description_map,
validate_media_description_map(opts[:description_map], opts[:language])},
{:ok, data} <- Upload.store(sanitize_upload_file(file), opts) do
obj_data = Maps.put_if_present(data, "actor", opts[:actor])
Repo.insert(%Object{data: obj_data})
else
{:description_map, _} -> {:error, dgettext("errors", "description_map invalid")}
e -> e
{:description_map, :invalid_language} ->
{:error, dgettext("errors", "valid language must be provided with description_map")}
{:description_map, _} ->
{:error, dgettext("errors", "description_map invalid")}
e ->
e
end
end

1
lib/pleroma/web/activity_pub/builder.ex
View file

@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.Builder do
alias Pleroma.Activity
alias Pleroma.Emoji
alias Pleroma.MultiLanguage
alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Relay

11
lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex
View file

@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
def history_awareness, do: :auto
def filter_by_summary(
%{data: %{"summaryMap" => %{} = parent_summary_map}} = _in_reply_to,
%{data: %{"summaryMap" => %{} = parent_summary_map} = parent},
%{"summaryMap" => %{} = child_summary_map} = child
) do
fixed_summary_map =
@ -25,9 +25,16 @@ def filter_by_summary(
end
end)
fixed_summary =
with {:ok, fixed} <- fix_one(child["summary"], parent["summary"]) do
fixed
else
_ -> child["summary"]
end
child
|> Map.put("summaryMap", fixed_summary_map)
|> Map.put("summary", Pleroma.MultiLanguage.map_to_str(fixed_summary_map, multiline: false))
|> Map.put("summary", fixed_summary)
end
def filter_by_summary(

5
lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex
View file

@ -103,10 +103,7 @@ def filter(
object
|> Map.put("contentMap", fixed_content_map)
|> Map.put(
"content",
Pleroma.MultiLanguage.map_to_str(fixed_content_map, multiline: true)
)
|> Map.put("content", fix_content(object["content"] || "", mention_users))
else
_ ->
# image-only posts from pleroma apparently reach this MRF without the content field

12
lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
View file

@ -88,16 +88,12 @@ defp check_replace(%{"object" => %{} = object} = message) do
config = Pleroma.Config.get([:mrf_keyword, :replace])
replace_kw = fn object ->
[
{"content", [multiline: true]},
{"name", [multiline: false]},
{"summary", [multiline: false]}
]
|> Enum.filter(fn {field, _} ->
["content", "name", "summary"]
|> Enum.filter(fn field ->
is_map(object[field <> "Map"]) or
(Map.has_key?(object, field) && object[field])
end)
|> Enum.reduce(object, fn {field, opts}, object ->
|> Enum.reduce(object, fn field, object ->
field_name_map = field <> "Map"
with %{} = data_map <- object[field_name_map] do
@ -108,7 +104,7 @@ defp check_replace(%{"object" => %{} = object} = message) do
object
|> Map.put(field_name_map, fixed_data_map)
|> Map.put(field, Pleroma.MultiLanguage.map_to_str(fixed_data_map, opts))
|> Map.put(field, replace_keyword(object[field], config))
else
_ ->
data = replace_keyword(object[field], config)

14
lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex
View file

@ -15,7 +15,7 @@ def history_awareness, do: :auto
def filter(
%{
"type" => type,
"object" => %{"contentMap" => %{} = content_map, "attachment" => _} = _child_object
"object" => %{"contentMap" => %{} = content_map, "attachment" => _} = child_object
} = object
)
when type in ["Create", "Update"] do
@ -28,6 +28,13 @@ def filter(
end
end)
fixed_content =
if child_object["content"] in @placeholders do
""
else
child_object["content"]
end
fixed_object =
if fixed_content_map == %{} do
Map.put(
@ -40,10 +47,7 @@ def filter(
else
object
|> put_in(["object", "contentMap"], fixed_content_map)
|> put_in(
["object", "content"],
Pleroma.MultiLanguage.map_to_str(fixed_content_map, multiline: true)
)
|> put_in(["object", "content"], fixed_content)
end
{:ok, fixed_object}

5
lib/pleroma/web/activity_pub/mrf/normalize_markup.ex
View file

@ -25,10 +25,7 @@ def filter(%{"type" => type, "object" => child_object} = object)
object
|> put_in(["object", "contentMap"], fixed_content_map)
|> put_in(
["object", "content"],
Pleroma.MultiLanguage.map_to_str(fixed_content_map, multiline: true)
)
|> put_in(["object", "content"], HTML.filter_tags(child_object["content"], scrub_policy))
else
_ ->
content =

3
lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
View file

@ -86,9 +86,6 @@ defp fix(data) do
|> fix_attachments()
|> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> CommonFixes.fix_multilang_field("content", "contentMap", multiline: true)
|> CommonFixes.fix_multilang_field("summary", "summaryMap", multiline: false)
|> CommonFixes.fix_multilang_field("name", "nameMap", multiline: false)
|> CommonFixes.maybe_add_language()
|> CommonFixes.maybe_add_content_map()
end

3
lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex
View file

@ -102,9 +102,6 @@ defp fix(data) do
|> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> fix_url()
|> CommonFixes.fix_multilang_field("content", "contentMap", multiline: true)
|> CommonFixes.fix_multilang_field("summary", "summaryMap", multiline: false)
|> CommonFixes.fix_multilang_field("name", "nameMap", multiline: false)
|> fix_content()
end

9
lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
View file

@ -194,13 +194,4 @@ def maybe_add_content_map(%{"language" => language, "content" => content} = obje
end
def maybe_add_content_map(object), do: object
def fix_multilang_field(data, str_field, map_field, opts \\ []) do
with %{} = map <- data[map_field],
str when is_binary(str) <- Pleroma.MultiLanguage.map_to_str(map, opts) do
Map.put(data, str_field, str)
else
_ -> data
end
end
end

9
lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex
View file

@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do
import Ecto.Changeset
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
@primary_key false
@ -25,15 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do
field(:type, :string, default: "Note")
end
defp fix(data) do
data
# name is used in Answers, so better not change it
|> CommonFixes.fix_multilang_field("nameRendered", "nameMap", multiline: false)
end
def changeset(struct, data) do
data = fix(data)
struct
|> cast(data, [:name, :nameRendered, :nameMap, :type])
|> cast_embed(:replies, with: &replies_changeset/2)

3
lib/pleroma/web/activity_pub/object_validators/question_validator.ex
View file

@ -67,9 +67,6 @@ defp fix(data) do
|> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> fix_closed()
|> CommonFixes.fix_multilang_field("content", "contentMap", multiline: true)
|> CommonFixes.fix_multilang_field("summary", "summaryMap", multiline: false)
|> CommonFixes.fix_multilang_field("name", "nameMap", multiline: false)
end
def changeset(struct, data) do

5
lib/pleroma/web/api_spec/operations/media_operation.ex
View file

@ -52,6 +52,11 @@ defp create_request do
type: :string,
description: "A plain-text description of the media, for accessibility purposes."
}),
language: %Schema{
type: :string,
nullable: true,
description: "ISO 639 language code for this status."
},
focus: %Schema{
type: :string,
description: "Two floating points (x,y), comma-delimited, ranging from -1.0 to 1.0."

10
lib/pleroma/web/common_api/activity_draft.ex
View file

@ -166,7 +166,11 @@ defp language(%{status: status} = draft) when is_binary(status) do
LanguageDetector.detect(draft.status <> " " <> (draft.summary || draft.params[:summary]))
if MultiLanguage.good_locale_code?(detected_language) do
%__MODULE__{draft | language: detected_language}
%__MODULE__{
draft
| params: Map.put(draft.params, :language, detected_language),
language: detected_language
}
else
draft
end
@ -529,9 +533,9 @@ defp validate(%{errors: [message | _]}), do: {:error, message}
defp differentiate_string_map(%{} = map), do: {nil, map}
defp differentiate_string_map(str) when is_binary(str), do: {str, nil}
defp get_source_map(%{status_map: %{} = status_map} = _draft) do
defp get_source_map(%{status_map: %{} = status_map} = draft) do
%{
"content" => Pleroma.MultiLanguage.map_to_str(status_map, mutiline: true),
"content" => Map.get(status_map, draft.language),
"contentMap" => status_map
}
end

2
lib/pleroma/web/common_api/utils.ex
View file

@ -167,7 +167,7 @@ def make_poll_data(%{poll: %{options_map: options_map, expires_in: expires_in}}
%{"name" => option["und"]}
else
%{
"name" => MultiLanguage.map_to_str(option, multiline: false),
"name" => Map.get(option, data.language),
"nameMap" => option
}
end

41
lib/pleroma/web/mastodon_api/controllers/media_controller.ex
View file

@ -5,6 +5,7 @@
defmodule Pleroma.Web.MastodonAPI.MediaController do
use Pleroma.Web, :controller
alias Pleroma.MultiLanguage
alias Pleroma.Object
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.Plugs.OAuthScopesPlug
@ -24,17 +25,25 @@ def create(
conn,
_
) do
with {:ok, object} <-
with language <- Map.get(data, :language),
{_, true} <-
{:valid_locale,
Map.get(data, :description_map) == nil or MultiLanguage.good_locale_code?(language)},
{:ok, object} <-
ActivityPub.upload(
file,
actor: user.ap_id,
description: Map.get(data, :description),
description_map: Map.get(data, :description_map)
description_map: Map.get(data, :description_map),
language: language
) do
attachment_data = Map.put(object.data, "id", object.id)
render(conn, "attachment.json", %{attachment: attachment_data})
else
{:valid_locale, _} ->
render_error(conn, 422, "valid language must be provided with description_map")
{:error, e} ->
conn
|> put_status(:unprocessable_entity)
@ -50,12 +59,17 @@ def create2(
conn,
_
) do
with {:ok, object} <-
with language <- Map.get(data, :language),
{_, true} <-
{:valid_locale,
Map.get(data, :description_map) == nil or MultiLanguage.good_locale_code?(language)},
{:ok, object} <-
ActivityPub.upload(
file,
actor: user.ap_id,
description: Map.get(data, :description),
description_map: Map.get(data, :description_map)
description_map: Map.get(data, :description_map),
language: language
) do
attachment_data = Map.put(object.data, "id", object.id)
@ -63,6 +77,9 @@ def create2(
|> put_status(202)
|> render("attachment.json", %{attachment: attachment_data})
else
{:valid_locale, _} ->
render_error(conn, 422, "valid language must be provided with description_map")
{:error, e} ->
conn
|> put_status(:unprocessable_entity)
@ -78,7 +95,7 @@ def update(
assigns: %{user: user},
private: %{
open_api_spex: %{
body_params: %{description_map: %{} = description_map},
body_params: %{description_map: %{} = description_map} = body_params,
params: %{id: id}
}
}
@ -87,18 +104,24 @@ def update(
) do
with %Object{} = object <- Object.get_by_id(id),
:ok <- Object.authorize_access(object, user),
{_, {:ok, %{}}} <-
{:description_map, Pleroma.MultiLanguage.validate_map(description_map)},
language = Map.get(body_params, :language, object["language"]),
{_, true} <-
{:valid_locale, description_map == nil or MultiLanguage.good_locale_code?(language)},
{_, {:ok, %{}}} <- {:description_map, MultiLanguage.validate_map(description_map)},
{:ok, %Object{data: data}} <-
Object.update_data(object, %{
"name" => Pleroma.MultiLanguage.map_to_str(description_map),
"name" => Map.get(description_map, language),
"nameMap" => description_map
}) do
attachment_data = Map.put(data, "id", object.id)
render(conn, "attachment.json", %{attachment: attachment_data})
else
{:description_map, _} -> render_error(conn, 422, "description_map not valid")
{:valid_locale, _} ->
render_error(conn, 422, "valid language must be provided with description_map")
{:description_map, _} ->
render_error(conn, 422, "description_map not valid")
end
end

36
test/pleroma/multi_language_test.exs
View file

@ -7,42 +7,6 @@ defmodule Pleroma.MultiLanguageTest do
alias Pleroma.MultiLanguage
describe "map_to_str" do
setup do
%{
data: %{
"en-US" => "mew",
"en-GB" => "meow"
}
}
end
test "single line", %{data: data} do
assert MultiLanguage.map_to_str(data) == "[en-GB] meow | [en-US] mew"
end
test "multi line", %{data: data} do
assert MultiLanguage.map_to_str(data, multiline: true) ==
"<div lang=\"en-GB\">meow</div><br><hr><br><div lang=\"en-US\">mew</div>"
end
test "only one language" do
data = %{"some" => "foo"}
assert MultiLanguage.map_to_str(data) == "foo"
assert MultiLanguage.map_to_str(data, multiline: true) == "foo"
end
test "resistent to tampering" do
data = %{
"en-US" => "mew {code} {content}",
"en-GB" => "meow {code} {content}"
}
assert MultiLanguage.map_to_str(data) ==
"[en-GB] meow {code} {content} | [en-US] mew {code} {content}"
end
end
describe "str_to_map" do
test "" do
assert MultiLanguage.str_to_map("foo") == %{"und" => "foo"}

5
test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs
View file

@ -56,12 +56,11 @@ test "it adds `re:` to summaryMap object when child summary and parent summary f
"c" => "another-object-summary"
}
assert res["object"]["summary"] ==
Pleroma.MultiLanguage.map_to_str(res["object"]["summaryMap"], multiline: false)
assert res["object"]["summary"] == "re: object-summary"
end
test "it adds `re:` to summary object when child summary contains re-subject of parent summary " do
s message = %{
message = %{
"type" => "Create",
"object" => %{
"summary" => "object-summary",

11
test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs
View file

@ -123,11 +123,10 @@ test "supports mulitlang" do
%{
"object" => %{
"content" => content,
"contentMap" =>
%{
"a" => content_a,
"b" => content_b
} = content_map
"contentMap" => %{
"a" => content_a,
"b" => content_b
}
}
}} = ForceMentionsInContent.filter(activity)
@ -136,7 +135,7 @@ test "supports mulitlang" do
assert content_a == mentions_part <> "mew mew"
assert content_b == mentions_part <> "lol lol"
assert content == Pleroma.MultiLanguage.map_to_str(content_map, multiline: true)
assert content == mentions_part <> "WHA-HA!"
end
test "don't mention self" do

23
test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs
View file

@ -318,23 +318,16 @@ test "replaces keyword in *Map" do
{:ok,
%{
"object" => %{
"content" => content,
"contentMap" =>
%{
"a" => "ZFS is free software",
"b" => "mew mew is also free software"
} = content_map,
"summary" => summary,
"summaryMap" =>
%{
"a" => "ZFS is very free software",
"b" => "mew mew is also very free software"
} = summary_map
"contentMap" => %{
"a" => "ZFS is free software",
"b" => "mew mew is also free software"
},
"summaryMap" => %{
"a" => "ZFS is very free software",
"b" => "mew mew is also very free software"
}
}
}} = KeywordPolicy.filter(message)
assert content == Pleroma.MultiLanguage.map_to_str(content_map, multiline: true)
assert summary == Pleroma.MultiLanguage.map_to_str(summary_map, multiline: false)
end
test "replaces keyword if string matches in history" do

2
test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs
View file

@ -32,7 +32,7 @@ test "multilang aware" do
}
assert {:ok, res} = NoPlaceholderTextPolicy.filter(message)
assert res["object"]["content"] == "lol"
assert res["object"]["content"] == ""
assert res["object"]["contentMap"] == %{"b" => "lol"}
message = %{

3
test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs
View file

@ -53,8 +53,7 @@ test "multilang-aware" do
assert {:ok, res} = NormalizeMarkup.filter(message)
assert res["object"]["contentMap"] == %{"a" => @expected, "b" => @expected}
assert res["object"]["content"] ==
Pleroma.MultiLanguage.map_to_str(res["object"]["contentMap"], multiline: true)
assert res["object"]["content"] == "some"
end
test "history-aware" do

5
test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
View file

@ -54,14 +54,9 @@ test "Note with contentMap and summaryMap", %{note: note} do
|> Map.put("summaryMap", summary_map)
|> Map.put("contentMap", content_map)
expected_summary = Pleroma.MultiLanguage.map_to_str(summary_map, multiline: false)
expected_content = Pleroma.MultiLanguage.map_to_str(content_map, multiline: true)
assert %{
valid?: true,
changes: %{
summary: ^expected_summary,
content: ^expected_content,
summaryMap: ^summary_map,
contentMap: ^content_map
}