Merge branch 'translateLocally' into 'main'

Support translateLocally translation provider

See merge request soapbox-pub/rebased!302

config/description.exs

@@ -3544,7 +3544,8 @@
         type: :module,
         suggestions: [
           Pleroma.Language.Translation.Deepl,
-          Pleroma.Language.Translation.Libretranslate
+          Pleroma.Language.Translation.Libretranslate,
+          Pleroma.Language.Translation.TranslateLocally
         ]
       },
       %{
@@ -3586,6 +3587,14 @@
         label: "LibreTranslate API Key",
         type: :string,
         suggestions: ["YOUR_API_KEY"]
+      },
+      %{
+        group: {:subgroup, Pleroma.Language.Translation.TranslateLocally},
+        key: :intermediate_language,
+        label:
+          "translateLocally intermediate language (used when direct source->target model is not available)",
+        type: :string,
+        suggestions: ["en"]
       }
     ]
   },

lib/pleroma/application_requirements.ex

@@ -200,10 +200,24 @@ defp check_system_commands!(:ok) do
         false
       end
 
+    translation_commands_status =
+      if Pleroma.Language.Translation.missing_dependencies() == [] do
+        true
+      else
+        Logger.error(
+          "The following dependencies required by the currently enabled " <>
+            "translation provider are not installed: " <>
+            inspect(Pleroma.Language.Translation.missing_dependencies())
+        )
+
+        false
+      end
+
     if Enum.all?(
          [
            preview_proxy_commands_status,
-           language_detector_commands_status
+           language_detector_commands_status,
+           translation_commands_status
            | filter_commands_statuses
          ],
          & &1

lib/pleroma/language/translation.ex

@@ -11,6 +11,16 @@ def configured? do
     !!provider and provider.configured?
   end
 
+  def missing_dependencies do
+    provider = get_provider()
+
+    if provider do
+      provider.missing_dependencies()
+    else
+      []
+    end
+  end
+
   def translate(text, source_language, target_language) do
     cache_key = get_cache_key(text, source_language, target_language)
 

lib/pleroma/language/translation/deepl.ex

@@ -7,6 +7,8 @@ defmodule Pleroma.Language.Translation.Deepl do
 
   alias Pleroma.Language.Translation.Provider
 
+  use Provider
+
   @behaviour Provider
 
   @name "DeepL"

lib/pleroma/language/translation/libretranslate.ex

@@ -7,6 +7,8 @@ defmodule Pleroma.Language.Translation.Libretranslate do
 
   alias Pleroma.Language.Translation.Provider
 
+  use Provider
+
   @behaviour Provider
 
   @name "LibreTranslate"

lib/pleroma/language/translation/provider.ex

@@ -3,6 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Language.Translation.Provider do
+  alias Pleroma.Language.Translation.Provider
+
+  @callback missing_dependencies() :: [String.t()]
+
   @callback configured?() :: boolean()
 
   @callback translate(
@@ -24,4 +28,13 @@ defmodule Pleroma.Language.Translation.Provider do
   @callback languages_matrix() :: {:ok, Map.t()} | {:error, atom()}
 
   @callback name() :: String.t()
+
+  defmacro __using__(_opts) do
+    quote do
+      @impl Provider
+      def missing_dependencies, do: []
+
+      defoverridable missing_dependencies: 0
+    end
+  end
 end

lib/pleroma/language/translation/translate_locally.ex

@@ -0,0 +1,129 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.Translation.TranslateLocally do
+  alias Pleroma.Language.Translation.Provider
+
+  use Provider
+
+  @behaviour Provider
+
+  @name "translateLocally"
+
+  @impl Provider
+  def missing_dependencies do
+    if Pleroma.Utils.command_available?("translateLocally") do
+      []
+    else
+      ["translateLocally"]
+    end
+  end
+
+  @impl Provider
+  def configured?, do: is_map(models())
+
+  @impl Provider
+  def translate(content, source_language, target_language) do
+    model =
+      models()
+      |> Map.get(source_language, %{})
+      |> Map.get(target_language)
+
+    models =
+      if model do
+        [model]
+      else
+        [
+          models()
+          |> Map.get(source_language, %{})
+          |> Map.get(intermediary_language()),
+          models()
+          |> Map.get(intermediary_language(), %{})
+          |> Map.get(target_language)
+        ]
+      end
+
+    translated_content =
+      Enum.reduce(models, content, fn model, content ->
+        text_path = Path.join(System.tmp_dir!(), "translateLocally-#{Ecto.UUID.generate()}")
+
+        File.write(text_path, content)
+
+        translated_content =
+          case System.cmd("translateLocally", ["-m", model, "-i", text_path, "--html"]) do
+            {content, _} -> content
+            _ -> nil
+          end
+
+        File.rm(text_path)
+
+        translated_content
+      end)
+
+    {:ok,
+     %{
+       content: translated_content,
+       detected_source_language: source_language,
+       provider: @name
+     }}
+  end
+
+  @impl Provider
+  def supported_languages(:source) do
+    languages =
+      languages_matrix()
+      |> elem(1)
+      |> Map.keys()
+
+    {:ok, languages}
+  end
+
+  @impl Provider
+  def supported_languages(:target) do
+    languages =
+      languages_matrix()
+      |> elem(1)
+      |> Map.values()
+      |> List.flatten()
+      |> Enum.uniq()
+
+    {:ok, languages}
+  end
+
+  @impl Provider
+  def languages_matrix do
+    languages =
+      models()
+      |> Map.to_list()
+      |> Enum.map(fn {key, value} -> {key, Map.keys(value)} end)
+      |> Enum.into(%{})
+
+    matrix =
+      if intermediary_language() do
+        languages
+        |> Map.to_list()
+        |> Enum.map(fn {key, value} ->
+          with_intermediary =
+            (((value ++ languages[intermediary_language()])
+              |> Enum.uniq()) --
+               [key])
+            |> Enum.sort()
+
+          {key, with_intermediary}
+        end)
+        |> Enum.into(%{})
+      else
+        languages
+      end
+
+    {:ok, matrix}
+  end
+
+  @impl Provider
+  def name, do: @name
+
+  defp models, do: Pleroma.Config.get([__MODULE__, :models])
+
+  defp intermediary_language, do: Pleroma.Config.get([__MODULE__, :intermediary_language])
+end

test/pleroma/language/translation/translate_locally_test.exs

@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Language.Translation.TranslateLocallyTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Language.Translation.TranslateLocally
+
+  @example_models %{
+    "de" => %{
+      "en" => "de-en-base"
+    },
+    "en" => %{
+      "de" => "en-de-base",
+      "pl" => "en-pl-tiny"
+    },
+    "cs" => %{
+      "en" => "cs-en-base"
+    },
+    "pl" => %{
+      "en" => "pl-en-tiny"
+    }
+  }
+
+  test "it returns languages list" do
+    clear_config([Pleroma.Language.Translation.TranslateLocally, :models], @example_models)
+
+    assert {:ok, languages} = TranslateLocally.supported_languages(:source)
+    assert ["cs", "de", "en", "pl"] = languages |> Enum.sort()
+  end
+
+  describe "it returns languages matrix" do
+    test "without intermediary language" do
+      clear_config([Pleroma.Language.Translation.TranslateLocally, :models], @example_models)
+
+      assert {:ok,
+              %{
+                "cs" => ["en"],
+                "de" => ["en"],
+                "en" => ["de", "pl"],
+                "pl" => ["en"]
+              }} = TranslateLocally.languages_matrix()
+    end
+
+    test "with intermediary language" do
+      clear_config([Pleroma.Language.Translation.TranslateLocally, :models], @example_models)
+      clear_config([Pleroma.Language.Translation.TranslateLocally, :intermediary_language], "en")
+
+      assert {:ok,
+              %{
+                "cs" => ["de", "en", "pl"],
+                "de" => ["en", "pl"],
+                "en" => ["de", "pl"],
+                "pl" => ["de", "en"]
+              }} = TranslateLocally.languages_matrix()
+    end
+  end
+end

test/support/translation_mock.ex

@@ -5,6 +5,8 @@
 defmodule TranslationMock do
   alias Pleroma.Language.Translation.Provider
 
+  use Provider
+
   @behaviour Provider
 
   @name "TranslationMock"