Add AntiDuplicationPolicy

config/config.exs

@@ -444,6 +444,10 @@
   reject_anonymous: true,
   reject_empty_message: true
 
+config :pleroma, :mrf_anti_duplication,
+  ttl: 60_000,
+  min_length: 50
+
 config :pleroma, :rich_media,
   enabled: true,
   ignore_hosts: [],

lib/pleroma/application.ex

@@ -214,6 +214,7 @@ defp cachex_children do
         expiration: chat_message_id_idempotency_key_expiration(),
         limit: 500_000
       ),
+      build_cachex("anti_duplication_mrf", limit: 5_000),
       build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000),
       build_cachex("rel_me", default_ttl: :timer.minutes(30), limit: 2_500)
     ]

lib/pleroma/caching.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Caching do
   # @callback del(Cachex.cache(), any(), Keyword.t()) :: {Cachex.status(), boolean()}
   @callback del(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
   @callback stream!(Cachex.cache(), any()) :: Enumerable.t()
+  @callback expire(Cachex.cache(), binary(), number()) :: {Cachex.status(), boolean()}
   @callback expire_at(Cachex.cache(), binary(), number()) :: {Cachex.status(), boolean()}
   @callback exists?(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
   @callback execute!(Cachex.cache(), function()) :: any()

lib/pleroma/web/activity_pub/mrf/anti_duplication_policy.ex

@@ -0,0 +1,45 @@
+defmodule Pleroma.Web.ActivityPub.MRF.AntiDuplicationPolicy do
+  @moduledoc "Prevents messages with the exact same content from being posted repeatedly, regardless of its source."
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @cache :anti_duplication_mrf_cache
+
+  @object_types ~w[Note Article Page ChatMessage Question Event]
+
+  @impl true
+  def filter(%{"type" => type, "content" => content} = object)
+      when is_binary(content) and type in @object_types do
+    ttl = Pleroma.Config.get([:mrf_anti_duplication, :ttl], :timer.minutes(1))
+    min_length = Pleroma.Config.get([:mrf_anti_duplication, :min_length], 50)
+
+    if String.length(content) >= min_length do
+      # We use SHA1 because it's faster and we don't need cryptographic security here.
+      key = :crypto.hash(:sha, content) |> Base.encode64(case: :lower)
+
+      case @cachex.exists?(@cache, key) do
+        {:ok, true} ->
+          @cachex.expire(@cache, key, ttl)
+          {:reject, "[AntiDuplicationPolicy] Message is a duplicate"}
+
+        _ ->
+          @cachex.put(@cache, key, true, ttl: ttl)
+          {:ok, object}
+      end
+    else
+      {:ok, object}
+    end
+  end
+
+  def filter(%{"object" => %{"type" => type, "content" => content} = object})
+      when is_binary(content) and type in @object_types do
+    filter(object)
+  end
+
+  def filter(object) do
+    {:ok, object}
+  end
+
+  @impl true
+  def describe, do: {:ok, %{}}
+end

test/pleroma/web/activity_pub/mrf/anti_duplication_policy_test.exs

@@ -0,0 +1,31 @@
+defmodule Pleroma.Web.ActivityPub.MRF.AntiDuplicationPolicyTest do
+  use Pleroma.DataCase
+  alias Pleroma.Web.ActivityPub.MRF.AntiDuplicationPolicy
+
+  test "prevents the same message twice" do
+    message = %{
+      "type" => "Create",
+      "object" => %{
+        "type" => "Note",
+        "content" =>
+          "In the beginning God created the heaven and the earth. And the earth was without form, and void; and darkness was upon the face of the deep."
+      }
+    }
+
+    {:ok, _} = AntiDuplicationPolicy.filter(message)
+    {:reject, _} = AntiDuplicationPolicy.filter(message)
+  end
+
+  test "allows short messages to be duplicated" do
+    message = %{
+      "type" => "Create",
+      "object" => %{
+        "type" => "Note",
+        "content" => "hello world"
+      }
+    }
+
+    {:ok, _} = AntiDuplicationPolicy.filter(message)
+    {:ok, _} = AntiDuplicationPolicy.filter(message)
+  end
+end

test/support/cachex_proxy.ex

@@ -26,6 +26,9 @@ defmodule Pleroma.CachexProxy do
   @impl true
   defdelegate fetch!(cache, key, func), to: Cachex
 
+  @impl true
+  defdelegate expire(cache, key, expiration), to: Cachex
+
   @impl true
   defdelegate expire_at(cache, str, num), to: Cachex
 

test/support/null_cache.ex

@@ -33,6 +33,9 @@ def get_and_update(_, _, func) do
     func.(nil)
   end
 
+  @impl true
+  def expire(_, _, _), do: {:ok, true}
+
   @impl true
   def expire_at(_, _, _), do: {:ok, true}