From 9e19753269a3d47f0cc711fd36d1a9301d63a9f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 13 Feb 2024 18:23:46 +0100 Subject: [PATCH 1/2] Support translateLocally translation provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/application_requirements.ex | 16 ++- lib/pleroma/language/translation.ex | 10 ++ lib/pleroma/language/translation/deepl.ex | 2 + .../language/translation/libretranslate.ex | 2 + lib/pleroma/language/translation/provider.ex | 13 ++ .../language/translation/translate_locally.ex | 117 ++++++++++++++++++ 6 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/language/translation/translate_locally.ex diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 295ba0e1af..ee4188fbce 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -200,10 +200,24 @@ defp check_system_commands!(:ok) do false end + translation_commands_status = + if Pleroma.Language.Translation.missing_dependencies() == [] do + true + else + Logger.error( + "The following dependencies required by the currently enabled " <> + "translation provider are not installed: " <> + inspect(Pleroma.Language.Translation.missing_dependencies()) + ) + + false + end + if Enum.all?( [ preview_proxy_commands_status, - language_detector_commands_status + language_detector_commands_status, + translation_commands_status | filter_commands_statuses ], & &1 diff --git a/lib/pleroma/language/translation.ex b/lib/pleroma/language/translation.ex index e4916389dd..be796802bc 100644 --- a/lib/pleroma/language/translation.ex +++ b/lib/pleroma/language/translation.ex @@ -11,6 +11,16 @@ def configured? do !!provider and provider.configured? end + def missing_dependencies do + provider = get_provider() + + if provider do + provider.missing_dependencies() + else + [] + end + end + def translate(text, source_language, target_language) do cache_key = get_cache_key(text, source_language, target_language) diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index 4f668fbba5..e027035b4d 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Language.Translation.Deepl do alias Pleroma.Language.Translation.Provider + use Provider + @behaviour Provider @name "DeepL" diff --git a/lib/pleroma/language/translation/libretranslate.ex b/lib/pleroma/language/translation/libretranslate.ex index b793b166e0..69ecf23b0b 100644 --- a/lib/pleroma/language/translation/libretranslate.ex +++ b/lib/pleroma/language/translation/libretranslate.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Language.Translation.Libretranslate do alias Pleroma.Language.Translation.Provider + use Provider + @behaviour Provider @name "LibreTranslate" diff --git a/lib/pleroma/language/translation/provider.ex b/lib/pleroma/language/translation/provider.ex index f12cba2cde..533b5355aa 100644 --- a/lib/pleroma/language/translation/provider.ex +++ b/lib/pleroma/language/translation/provider.ex @@ -3,6 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Language.Translation.Provider do + alias Pleroma.Language.Translation.Provider + + @callback missing_dependencies() :: [String.t()] + @callback configured?() :: boolean() @callback translate( @@ -24,4 +28,13 @@ defmodule Pleroma.Language.Translation.Provider do @callback languages_matrix() :: {:ok, Map.t()} | {:error, atom()} @callback name() :: String.t() + + defmacro __using__(_opts) do + quote do + @impl Provider + def missing_dependencies, do: [] + + defoverridable missing_dependencies: 0 + end + end end diff --git a/lib/pleroma/language/translation/translate_locally.ex b/lib/pleroma/language/translation/translate_locally.ex new file mode 100644 index 0000000000..4a3e9d7acc --- /dev/null +++ b/lib/pleroma/language/translation/translate_locally.ex @@ -0,0 +1,117 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.Translation.TranslateLocally do + alias Pleroma.Language.Translation.Provider + + use Provider + + @behaviour Provider + + @name "translateLocally" + + @impl Provider + def missing_dependencies do + if Pleroma.Utils.command_available?("translateLocally") do + [] + else + ["translateLocally"] + end + end + + @impl Provider + def configured?, do: is_map(models()) + + @impl Provider + def translate(content, source_language, target_language) do + model = + models() + |> Map.get(source_language, %{}) + |> Map.get(target_language) + + models = + if model do + [model] + else + [ + models() + |> Map.get(source_language, %{}) + |> Map.get(intermediary_language()), + models() + |> Map.get(intermediary_language(), %{}) + |> Map.get(target_language) + ] + end + + translated_content = + Enum.reduce(models, content, fn model, content -> + text_path = Path.join(System.tmp_dir!(), "translateLocally-#{Ecto.UUID.generate()}") + + File.write(text_path, content) + + translated_content = + case System.cmd("translateLocally", ["-m", model, "-i", text_path, "--html"]) do + {content, _} -> content + _ -> nil + end + + File.rm(text_path) + + translated_content + end) + + {:ok, + %{ + content: translated_content, + detected_source_language: source_language, + provider: @name + }} + end + + @impl Provider + def supported_languages(:source) do + languages_matrix() + |> Map.keys() + end + + @impl Provider + def supported_languages(:target) do + languages_matrix() + |> Map.values() + |> List.flatten() + |> Enum.uniq() + end + + @impl Provider + def languages_matrix do + languages = + models() + |> Map.to_list() + |> Enum.map(fn {key, value} -> {key, Map.keys(value)} end) + |> Enum.into(%{}) + + if intermediary_language() do + languages + |> Map.to_list() + |> Enum.map(fn {key, value} -> + with_intermediary = + ((value ++ languages[intermediary_language()]) + |> Enum.uniq()) -- + [intermediary_language()] + + {key, with_intermediary} + end) + |> Enum.into(%{}) + else + languages + end + end + + @impl Provider + def name, do: @name + + defp models, do: Pleroma.Config.get([__MODULE__, :models]) + + defp intermediary_language, do: Pleroma.Config.get([__MODULE__, :intermediary_language]) +end From eb1995d7c6584b914c893c431933f9f39e4ef373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 13 Feb 2024 20:08:17 +0100 Subject: [PATCH 2/2] Add test for translateLocally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 11 +++- .../language/translation/translate_locally.ex | 54 ++++++++++------- .../translation/translate_locally_test.exs | 59 +++++++++++++++++++ test/support/translation_mock.ex | 2 + 4 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 test/pleroma/language/translation/translate_locally_test.exs diff --git a/config/description.exs b/config/description.exs index 33d959aab4..73ca4809b3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3544,7 +3544,8 @@ type: :module, suggestions: [ Pleroma.Language.Translation.Deepl, - Pleroma.Language.Translation.Libretranslate + Pleroma.Language.Translation.Libretranslate, + Pleroma.Language.Translation.TranslateLocally ] }, %{ @@ -3586,6 +3587,14 @@ label: "LibreTranslate API Key", type: :string, suggestions: ["YOUR_API_KEY"] + }, + %{ + group: {:subgroup, Pleroma.Language.Translation.TranslateLocally}, + key: :intermediate_language, + label: + "translateLocally intermediate language (used when direct source->target model is not available)", + type: :string, + suggestions: ["en"] } ] }, diff --git a/lib/pleroma/language/translation/translate_locally.ex b/lib/pleroma/language/translation/translate_locally.ex index 4a3e9d7acc..7eaa95e7b2 100644 --- a/lib/pleroma/language/translation/translate_locally.ex +++ b/lib/pleroma/language/translation/translate_locally.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors +# Copyright © 2017-2024 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Language.Translation.TranslateLocally do @@ -71,16 +71,24 @@ def translate(content, source_language, target_language) do @impl Provider def supported_languages(:source) do - languages_matrix() - |> Map.keys() + languages = + languages_matrix() + |> elem(1) + |> Map.keys() + + {:ok, languages} end @impl Provider def supported_languages(:target) do - languages_matrix() - |> Map.values() - |> List.flatten() - |> Enum.uniq() + languages = + languages_matrix() + |> elem(1) + |> Map.values() + |> List.flatten() + |> Enum.uniq() + + {:ok, languages} end @impl Provider @@ -91,21 +99,25 @@ def languages_matrix do |> Enum.map(fn {key, value} -> {key, Map.keys(value)} end) |> Enum.into(%{}) - if intermediary_language() do - languages - |> Map.to_list() - |> Enum.map(fn {key, value} -> - with_intermediary = - ((value ++ languages[intermediary_language()]) - |> Enum.uniq()) -- - [intermediary_language()] + matrix = + if intermediary_language() do + languages + |> Map.to_list() + |> Enum.map(fn {key, value} -> + with_intermediary = + (((value ++ languages[intermediary_language()]) + |> Enum.uniq()) -- + [key]) + |> Enum.sort() - {key, with_intermediary} - end) - |> Enum.into(%{}) - else - languages - end + {key, with_intermediary} + end) + |> Enum.into(%{}) + else + languages + end + + {:ok, matrix} end @impl Provider diff --git a/test/pleroma/language/translation/translate_locally_test.exs b/test/pleroma/language/translation/translate_locally_test.exs new file mode 100644 index 0000000000..51cbd11bd8 --- /dev/null +++ b/test/pleroma/language/translation/translate_locally_test.exs @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.Translation.TranslateLocallyTest do + use Pleroma.DataCase + + alias Pleroma.Language.Translation.TranslateLocally + + @example_models %{ + "de" => %{ + "en" => "de-en-base" + }, + "en" => %{ + "de" => "en-de-base", + "pl" => "en-pl-tiny" + }, + "cs" => %{ + "en" => "cs-en-base" + }, + "pl" => %{ + "en" => "pl-en-tiny" + } + } + + test "it returns languages list" do + clear_config([Pleroma.Language.Translation.TranslateLocally, :models], @example_models) + + assert {:ok, languages} = TranslateLocally.supported_languages(:source) + assert ["cs", "de", "en", "pl"] = languages |> Enum.sort() + end + + describe "it returns languages matrix" do + test "without intermediary language" do + clear_config([Pleroma.Language.Translation.TranslateLocally, :models], @example_models) + + assert {:ok, + %{ + "cs" => ["en"], + "de" => ["en"], + "en" => ["de", "pl"], + "pl" => ["en"] + }} = TranslateLocally.languages_matrix() + end + + test "with intermediary language" do + clear_config([Pleroma.Language.Translation.TranslateLocally, :models], @example_models) + clear_config([Pleroma.Language.Translation.TranslateLocally, :intermediary_language], "en") + + assert {:ok, + %{ + "cs" => ["de", "en", "pl"], + "de" => ["en", "pl"], + "en" => ["de", "pl"], + "pl" => ["de", "en"] + }} = TranslateLocally.languages_matrix() + end + end +end diff --git a/test/support/translation_mock.ex b/test/support/translation_mock.ex index 95da738d10..84ed8f696e 100644 --- a/test/support/translation_mock.ex +++ b/test/support/translation_mock.ex @@ -5,6 +5,8 @@ defmodule TranslationMock do alias Pleroma.Language.Translation.Provider + use Provider + @behaviour Provider @name "TranslationMock"