Improve getting language from context, add more tests

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -532,6 +532,7 @@ def handle_incoming(
       |> fix_type(fetch_options)
       |> fix_in_reply_to(fetch_options)
       |> fix_quote_url(fetch_options)
+      |> maybe_add_language_from_activity(data)
 
     data = Map.put(data, "object", object)
     options = Keyword.put(options, :local, false)
@@ -1113,6 +1114,16 @@ def maybe_add_language(object) do
     end
   end
 
+  def maybe_add_language_from_activity(object, activity) do
+    language = get_language_from_context(activity) |> get_valid_language()
+
+    if language do
+      Map.put(object, "language", language)
+    else
+      object
+    end
+  end
+
   defp get_language_from_context(%{"@context" => context}) when is_list(context) do
     case context
          |> Enum.find(fn
@@ -1137,8 +1148,8 @@ defp get_language_from_content_map(%{"contentMap" => content_map, "content" => s
 
   defp get_language_from_content_map(_), do: nil
 
-  defp get_language_from_content(%{"summary" => summary, "content" => content}) do
-    LanguageDetector.detect("#{summary} #{content}")
+  defp get_language_from_content(%{"content" => content}) do
+    LanguageDetector.detect(content)
   end
 
   defp get_language_from_content(_), do: nil

test/fixtures/tesla_mock/deepl-translation.json

@@ -0,0 +1 @@
+{"translations":[{"detected_source_language":"PL","text":"REMOVE THE FOLLOWER!Paste this on your follower. If we get 70% of nk users...they will remove the follower!!!"}]}

test/pleroma/language/translation/deepl.ex

@@ -0,0 +1,27 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.TranslationTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Language.Translation.Deepl
+
+  test "it translates text" do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    clear_config([Pleroma.Language.Translation.Deepl, :plan], :free)
+    clear_config([Pleroma.Language.Translation.Deepl, :api_key], "API_KEY")
+
+    {:ok, res} =
+             Deepl.translate(
+               "USUNĄĆ ŚLEDZIKA!Wklej to na swojego śledzika. Jeżeli uzbieramy 70% użytkowników nk...to usuną śledzika!!!",
+               "pl",
+               "en"
+             )
+
+    assert %{
+             detected_source_language: "PL",
+             provider: "DeepL"
+           } = res
+  end
+end

test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs

@@ -385,6 +385,87 @@ test "it correctly processes messages with weirdness in address fields" do
       assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"]
       assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"]
     end
+
+    test "it detects language from context" do
+      user = insert(:user)
+
+      message = %{
+        "@context" => ["https://www.w3.org/ns/activitystreams", %{"@language" => "pl"}],
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "cc" => [],
+        "type" => "Create",
+        "object" => %{
+          "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+          "cc" => [],
+          "id" => Utils.generate_object_id(),
+          "type" => "Note",
+          "content" => "Szczęść Boże",
+          "attributedTo" => user.ap_id
+        },
+        "actor" => user.ap_id
+      }
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+      object = Object.normalize(data["object"], fetch: false)
+
+      assert object.data["language"] == "pl"
+    end
+
+    test "it detects language from contentMap" do
+      user = insert(:user)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "cc" => [],
+        "type" => "Create",
+        "object" => %{
+          "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+          "cc" => [],
+          "id" => Utils.generate_object_id(),
+          "type" => "Note",
+          "content" => "Szczęść Boże",
+          "contentMap" => %{
+            "de" => "Gott segne",
+            "pl" => "Szczęść Boże"
+          },
+          "attributedTo" => user.ap_id
+        },
+        "actor" => user.ap_id
+      }
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+      object = Object.normalize(data["object"], fetch: false)
+
+      assert object.data["language"] == "pl"
+    end
+
+    test "it detects language from content" do
+      clear_config([Pleroma.Language.LanguageDetector, :provider], LanguageDetectorMock)
+
+      user = insert(:user)
+
+      message = %{
+        "@context" => ["https://www.w3.org/ns/activitystreams"],
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "cc" => [],
+        "type" => "Create",
+        "object" => %{
+          "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+          "cc" => [],
+          "id" => Utils.generate_object_id(),
+          "type" => "Note",
+          "content" => "Dieu vous bénisse, Fédivers.",
+          "attributedTo" => user.ap_id
+        },
+        "actor" => user.ap_id
+      }
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+      object = Object.normalize(data["object"], fetch: false)
+
+      assert object.data["language"] == "fr"
+    end
   end
 
   describe "`handle_incoming/2`, Mastodon format `replies` handling" do

test/support/http_request_mock.ex

@@ -1560,6 +1560,15 @@ def post("http://404.site" <> _, _, _, _) do
      }}
   end
 
+  def post("https://api-free.deepl.com/v2/translate" <> _, _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/deepl-translation.json"),
+       headers: [{"content-type", "application/json"}]
+     }}
+  end
+
   def post(url, query, body, headers) do
     {:error,
      "Mock response not implemented for POST #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}