From c899af1d6acad1895240a0247e9b91eca5db08df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 14 Apr 2022 20:09:43 +0200 Subject: [PATCH 01/61] Reject requests from specified instances if `authorized_fetch_mode` is enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/config.exs | 1 + config/description.exs | 12 ++++ docs/configuration/cheatsheet.md | 1 + lib/pleroma/signature.ex | 16 +++-- .../web/mastodon_api/views/instance_view.ex | 7 +++ lib/pleroma/web/plugs/http_signature_plug.ex | 40 ++++++++++++ test/pleroma/signature_test.exs | 8 +++ .../web/plugs/http_signature_plug_test.exs | 63 +++++++++++++++++-- 8 files changed, 140 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index 0fc9598077..cfa16f766c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -216,6 +216,7 @@ allow_relay: true, public: true, quarantined_instances: [], + rejected_instances: [], static_dir: "instance/static/", allowed_post_formats: [ "text/plain", diff --git a/config/description.exs b/config/description.exs index b29348edfc..a75f409292 100644 --- a/config/description.exs +++ b/config/description.exs @@ -714,6 +714,18 @@ {"*.quarantined.com", "Reason"} ] }, + %{ + key: :rejected_instances, + type: {:list, :tuple}, + key_placeholder: "instance", + value_placeholder: "reason", + description: + "List of ActivityPub instances to reject requests from if authorized_fetch_mode is enabled", + suggestions: [ + {"rejected.com", "Reason"}, + {"*.rejected.com", "Reason"} + ] + }, %{ key: :static_dir, type: :string, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 6e13b96223..84a5bdb98e 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -41,6 +41,7 @@ To add configuration to your config file, you can copy it from the base config. * `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance. * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details. * `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send. +* `rejected_instances`: ActivityPub instances to reject requests from if authorized_fetch_mode is enabled. * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML). * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with older software for theses nicknames. diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index dbe6fd209f..d5ba5c4fb6 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -37,8 +37,7 @@ def key_id_to_actor_id(key_id) do end def fetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else @@ -48,8 +47,7 @@ def fetch_public_key(conn) do end def refetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} @@ -59,6 +57,16 @@ def refetch_public_key(conn) do end end + def get_actor_id(conn) do + with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), + {:ok, actor_id} <- key_id_to_actor_id(kid) do + {:ok, actor_id} + else + e -> + {:error, e} + end + end + def sign(%User{} = user, headers) do with {:ok, %{keys: keys}} <- User.ensure_keys_present(user), {:ok, private_key, _} <- Keys.keys_from_pem(keys) do diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 62931bd41b..017bd62e2f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -105,6 +105,7 @@ def features do def federation do quarantined = Config.get([:instance, :quarantined_instances], []) + rejected = Config.get([:instance, :rejected_instances], []) if Config.get([:mrf, :transparency]) do {:ok, data} = MRF.describe() @@ -124,6 +125,12 @@ def federation do |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) |> Map.new() }) + |> Map.put( + :rejected_instances, + rejected + |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) + |> Map.new() + ) else %{} end diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index d023754a65..cf80b9b147 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -5,6 +5,10 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] + + alias Pleroma.Config + alias Pleroma.Web.ActivityPub.MRF + require Logger def init(options) do @@ -19,7 +23,9 @@ def call(conn, _opts) do if get_format(conn) == "activity+json" do conn |> maybe_assign_valid_signature() + |> maybe_assign_actor_id() |> maybe_require_signature() + |> maybe_filter_requests() else conn end @@ -46,6 +52,16 @@ defp maybe_assign_valid_signature(conn) do end end + defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do + adapter = Application.get_env(:http_signatures, :adapter) + + {:ok, actor_id} = adapter.get_actor_id(conn) + + assign(conn, :actor_id, actor_id) + end + + defp maybe_assign_actor_id(conn), do: conn + defp has_signature_header?(conn) do conn |> get_req_header("signature") |> Enum.at(0, false) end @@ -62,4 +78,28 @@ defp maybe_require_signature(conn) do conn end end + + defp maybe_filter_requests(%{halted: true} = conn), do: conn + + defp maybe_filter_requests(conn) do + if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + %{host: host} = URI.parse(conn.assigns.actor_id) + + if MRF.subdomain_match?(rejected_domains(), host) do + conn + |> put_status(:unauthorized) + |> halt() + else + conn + end + else + conn + end + end + + defp rejected_domains do + Config.get([:instance, :rejected_instances]) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + end end diff --git a/test/pleroma/signature_test.exs b/test/pleroma/signature_test.exs index 92d05f26cd..8f94efdc33 100644 --- a/test/pleroma/signature_test.exs +++ b/test/pleroma/signature_test.exs @@ -70,6 +70,14 @@ test "it returns error when not found user" do end end + describe "get_actor_id/1" do + test "it returns actor id" do + ap_id = "https://mastodon.social/users/lambadalambda" + + assert Signature.get_actor_id(make_fake_conn(ap_id)) == {:ok, ap_id} + end + end + describe "sign/2" do test "it returns signature headers" do user = diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 2d8fba3cd2..de68e8823a 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -10,11 +10,15 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do import Phoenix.Controller, only: [put_format: 2] import Mock - test "it call HTTPSignatures to check validity if the actor sighed it" do + test "it call HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) - with_mock HTTPSignatures, validate_conn: fn _ -> true end do + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do conn = conn |> put_req_header( @@ -41,7 +45,11 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do end test "when signature header is present", %{conn: conn} do - with_mock HTTPSignatures, validate_conn: fn _ -> false end do + with_mock HTTPSignatures, + validate_conn: fn _ -> false end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do conn = conn |> put_req_header( @@ -58,7 +66,11 @@ test "when signature header is present", %{conn: conn} do assert called(HTTPSignatures.validate_conn(:_)) end - with_mock HTTPSignatures, validate_conn: fn _ -> true end do + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do conn = conn |> put_req_header( @@ -82,4 +94,47 @@ test "halts the connection when `signature` header is not present", %{conn: conn assert conn.resp_body == "Request not signed" end end + + test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do + clear_config([:activitypub, :authorized_fetch_mode], true) + clear_config([:instance, :rejected_instances], [{"mastodon.example.org", "no reason"}]) + + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == true + assert called(HTTPSignatures.validate_conn(:_)) + end + + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://allowed.example.org/users/admin#main-key"} + end do + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://allowed.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false + assert called(HTTPSignatures.validate_conn(:_)) + end + end end From 2ae1b802f260e9ad8eaa585907d9505545ceb872 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Mar 2023 10:21:11 +0100 Subject: [PATCH 02/61] AttachmentValidator: Add support for Honk "summary" + "name" As used by Honk and supported by Mastodon --- .../object_validators/attachment_validator.ex | 3 ++- .../object_validators/attachment_validator_test.exs | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 398020bff1..766421e602 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string) field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) + field(:summary, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do @@ -44,7 +45,7 @@ def changeset(struct, data) do |> fix_url() struct - |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash]) |> cast_embed(:url, with: &url_changeset/2, required: true) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType]) diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs index 77f2044e97..8d561603c0 100644 --- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs @@ -25,19 +25,22 @@ test "fails without url" do end test "works with honkerific attachments" do - attachment = %{ + honk = %{ "mediaType" => "", - "name" => "", - "summary" => "298p3RG7j27tfsZ9RQ.jpg", + "summary" => "Select your spirit chonk", + "name" => "298p3RG7j27tfsZ9RQ.jpg", "type" => "Document", "url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg" } assert {:ok, attachment} = - AttachmentValidator.cast_and_validate(attachment) + honk + |> AttachmentValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) assert attachment.mediaType == "application/octet-stream" + assert attachment.summary == "Select your spirit chonk" + assert attachment.name == "298p3RG7j27tfsZ9RQ.jpg" end test "works with an unknown but valid mime type" do From 197647a04e66c1af3ae691a4507612fdbee9c48c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Mar 2023 10:35:57 +0100 Subject: [PATCH 03/61] MastoAPI Attachment: Use "summary" for descriptions if present --- .../web/api_spec/schemas/attachment.ex | 6 +- .../web/mastodon_api/views/status_view.ex | 17 ++- .../mastodon_api/views/status_view_test.exs | 101 ++++++++++++------ 3 files changed, 87 insertions(+), 37 deletions(-) diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex index 48634a14f3..e89f2ddd06 100644 --- a/lib/pleroma/web/api_spec/schemas/attachment.ex +++ b/lib/pleroma/web/api_spec/schemas/attachment.ex @@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do pleroma: %Schema{ type: :object, properties: %{ - mime_type: %Schema{type: :string, description: "mime type of the attachment"} + mime_type: %Schema{type: :string, description: "mime type of the attachment"}, + name: %Schema{ + type: :string, + description: "Name of the attachment, typically the filename" + } } } }, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0a8c98b442..7a3af8acb3 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -571,6 +571,19 @@ def render("attachment.json", %{attachment: attachment}) do to_string(attachment["id"] || hash_id) end + description = + if attachment["summary"] do + HTML.strip_tags(attachment["summary"]) + else + attachment["name"] + end + + name = if attachment["summary"], do: attachment["name"] + + pleroma = + %{mime_type: media_type} + |> Maps.put_if_present(:name, name) + %{ id: attachment_id, url: href, @@ -578,8 +591,8 @@ def render("attachment.json", %{attachment: attachment}) do preview_url: href_preview, text_url: href, type: type, - description: attachment["name"], - pleroma: %{mime_type: media_type}, + description: description, + pleroma: pleroma, blurhash: attachment["blurhash"] } |> Maps.put_if_present(:meta, meta) diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index f76b115b7a..4580da75b2 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -456,45 +456,78 @@ test "create mentions from the 'tag' field" do assert mention.url == recipient.ap_id end - test "attachments" do - object = %{ - "type" => "Image", - "url" => [ - %{ - "mediaType" => "image/png", - "href" => "someurl", - "width" => 200, - "height" => 100 - } - ], - "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", - "uuid" => 6 - } + describe "attachments" do + test "Complete Mastodon style" do + object = %{ + "type" => "Image", + "url" => [ + %{ + "mediaType" => "image/png", + "href" => "someurl", + "width" => 200, + "height" => 100 + } + ], + "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", + "uuid" => 6 + } - expected = %{ - id: "1638338801", - type: "image", - url: "someurl", - remote_url: "someurl", - preview_url: "someurl", - text_url: "someurl", - description: nil, - pleroma: %{mime_type: "image/png"}, - meta: %{original: %{width: 200, height: 100, aspect: 2}}, - blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" - } + expected = %{ + id: "1638338801", + type: "image", + url: "someurl", + remote_url: "someurl", + preview_url: "someurl", + text_url: "someurl", + description: nil, + pleroma: %{mime_type: "image/png"}, + meta: %{original: %{width: 200, height: 100, aspect: 2}}, + blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" + } - api_spec = Pleroma.Web.ApiSpec.spec() + api_spec = Pleroma.Web.ApiSpec.spec() - assert expected == StatusView.render("attachment.json", %{attachment: object}) - assert_schema(expected, "Attachment", api_spec) + assert expected == StatusView.render("attachment.json", %{attachment: object}) + assert_schema(expected, "Attachment", api_spec) - # If theres a "id", use that instead of the generated one - object = Map.put(object, "id", 2) - result = StatusView.render("attachment.json", %{attachment: object}) + # If theres a "id", use that instead of the generated one + object = Map.put(object, "id", 2) + result = StatusView.render("attachment.json", %{attachment: object}) - assert %{id: "2"} = result - assert_schema(result, "Attachment", api_spec) + assert %{id: "2"} = result + assert_schema(result, "Attachment", api_spec) + end + + test "Honkerific" do + object = %{ + "type" => "Image", + "url" => [ + %{ + "mediaType" => "image/png", + "href" => "someurl" + } + ], + "name" => "fool.jpeg", + "summary" => "they have played us for absolute fools." + } + + expected = %{ + blurhash: nil, + description: "they have played us for absolute fools.", + id: "1638338801", + pleroma: %{mime_type: "image/png", name: "fool.jpeg"}, + preview_url: "someurl", + remote_url: "someurl", + text_url: "someurl", + type: "image", + url: "someurl" + } + + api_spec = Pleroma.Web.ApiSpec.spec() + + assert expected == StatusView.render("attachment.json", %{attachment: object}) + assert_schema(expected, "Attachment", api_spec) + end end test "put the url advertised in the Activity in to the url attribute" do From 1ab4ab8d38687634735e1415f395b072718ab1ab Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 18 Jul 2023 18:24:30 -0400 Subject: [PATCH 04/61] Extract translatable strings --- changelog.d/3907.skip | 0 priv/gettext/config_descriptions.pot | 84 ++++++++++++++++++++++++++++ priv/gettext/errors.pot | 36 +++++++++--- priv/gettext/oauth_scopes.pot | 40 +++++++++++++ 4 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 changelog.d/3907.skip diff --git a/changelog.d/3907.skip b/changelog.d/3907.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/priv/gettext/config_descriptions.pot b/priv/gettext/config_descriptions.pot index 4f60e1c854..b4792868b6 100644 --- a/priv/gettext/config_descriptions.pot +++ b/priv/gettext/config_descriptions.pot @@ -5973,3 +5973,87 @@ msgstr "" msgctxt "config label at :pleroma-:instance > :languages" msgid "Languages" msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji" +msgid "Reject or force-unlisted emojis whose URLs or names match a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html)." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode" +msgid " A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_url" +msgid " A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :remove_shortcode" +msgid " A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :remove_url" +msgid " A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_chunk_size" +msgid "The number of activities to fetch in the backup job for each chunk." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_wait_time" +msgid "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji" +msgid "MRF Emoji" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode" +msgid "Federated timeline removal shortcode" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_url" +msgid "Federated timeline removal url" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :remove_shortcode" +msgid "Remove shortcode" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :remove_url" +msgid "Remove url" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_chunk_size" +msgid "Process Chunk Size" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_wait_time" +msgid "Process Wait Time" +msgstr "" diff --git a/priv/gettext/errors.pot b/priv/gettext/errors.pot index d320ee1bdd..aca77f8fa6 100644 --- a/priv/gettext/errors.pot +++ b/priv/gettext/errors.pot @@ -110,7 +110,7 @@ msgstr "" msgid "Can't display this activity" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:334 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:346 #, elixir-autogen, elixir-format msgid "Can't find user" msgstr "" @@ -198,7 +198,7 @@ msgstr "" msgid "Invalid password." msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:279 #, elixir-autogen, elixir-format msgid "Invalid request" msgstr "" @@ -225,7 +225,7 @@ msgstr "" #: lib/pleroma/web/feed/tag_controller.ex:16 #: lib/pleroma/web/feed/user_controller.ex:69 #: lib/pleroma/web/o_status/o_status_controller.ex:132 -#: lib/pleroma/web/plugs/uploaded_media.ex:104 +#: lib/pleroma/web/plugs/uploaded_media.ex:84 #, elixir-autogen, elixir-format msgid "Not found" msgstr "" @@ -235,7 +235,7 @@ msgstr "" msgid "Poll's author can't vote" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:499 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:511 #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:39 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:51 @@ -341,7 +341,7 @@ msgstr "" msgid "CAPTCHA expired" msgstr "" -#: lib/pleroma/web/plugs/uploaded_media.ex:77 +#: lib/pleroma/web/plugs/uploaded_media.ex:57 #, elixir-autogen, elixir-format msgid "Failed" msgstr "" @@ -361,7 +361,7 @@ msgstr "" msgid "Insufficient permissions: %{permissions}." msgstr "" -#: lib/pleroma/web/plugs/uploaded_media.ex:131 +#: lib/pleroma/web/plugs/uploaded_media.ex:111 #, elixir-autogen, elixir-format msgid "Internal Error" msgstr "" @@ -557,7 +557,7 @@ msgstr "" msgid "Access denied" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:331 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:343 #, elixir-autogen, elixir-format msgid "This API requires an authenticated user" msgstr "" @@ -567,7 +567,7 @@ msgstr "" msgid "User is not an admin." msgstr "" -#: lib/pleroma/user/backup.ex:73 +#: lib/pleroma/user/backup.ex:78 #, elixir-format msgid "Last export was less than a day ago" msgid_plural "Last export was less than %{days} days ago" @@ -607,3 +607,23 @@ msgstr "" #, elixir-autogen, elixir-format msgid "User isn't privileged." msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267 +#, elixir-autogen, elixir-format +msgid "Bio is too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:270 +#, elixir-autogen, elixir-format +msgid "Name is too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:273 +#, elixir-autogen, elixir-format +msgid "One or more field entries are too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:276 +#, elixir-autogen, elixir-format +msgid "Too many field entries" +msgstr "" diff --git a/priv/gettext/oauth_scopes.pot b/priv/gettext/oauth_scopes.pot index 50ad0dd9ed..83328770e5 100644 --- a/priv/gettext/oauth_scopes.pot +++ b/priv/gettext/oauth_scopes.pot @@ -219,3 +219,43 @@ msgstr "" #, elixir-autogen, elixir-format msgid "read:mutes" msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "push" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:backups" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:chats" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:media" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:reports" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:chats" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:follow" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:reports" +msgstr "" From f271ea6e432d685c113582e5944d79e12c153016 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 16 Dec 2023 18:22:32 +0100 Subject: [PATCH 05/61] Move Plugs.RemoteIP.maybe_add_cidr/1 to InetHelper.parse_cidr/1 --- lib/pleroma/helpers/inet_helper.ex | 11 +++++++++++ lib/pleroma/web/plugs/remote_ip.ex | 14 ++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex index 704d37f8a7..3500fc6791 100644 --- a/lib/pleroma/helpers/inet_helper.ex +++ b/lib/pleroma/helpers/inet_helper.ex @@ -16,4 +16,15 @@ def parse_address(ip) when is_binary(ip) do def parse_address(ip) do :inet.parse_address(ip) end + + def parse_cidr(proxy) when is_binary(proxy) do + proxy = + cond do + "/" in String.codepoints(proxy) -> proxy + InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" + InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" + end + + InetCidr.parse(proxy, true) + end end diff --git a/lib/pleroma/web/plugs/remote_ip.ex b/lib/pleroma/web/plugs/remote_ip.ex index f207d9fef9..3a4bffb50d 100644 --- a/lib/pleroma/web/plugs/remote_ip.ex +++ b/lib/pleroma/web/plugs/remote_ip.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do """ alias Pleroma.Config + alias Pleroma.Helpers.InetHelper import Plug.Conn @behaviour Plug @@ -30,19 +31,8 @@ defp remote_ip_opts do proxies = Config.get([__MODULE__, :proxies], []) |> Enum.concat(reserved) - |> Enum.map(&maybe_add_cidr/1) + |> Enum.map(&InetHelper.parse_cidr/1) {headers, proxies} end - - defp maybe_add_cidr(proxy) when is_binary(proxy) do - proxy = - cond do - "/" in String.codepoints(proxy) -> proxy - InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" - InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" - end - - InetCidr.parse(proxy, true) - end end From 086ba59d0346be870dc7df2660fbb55666bf0af7 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 16 Dec 2023 18:56:46 +0100 Subject: [PATCH 06/61] HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions --- changelog.d/auth-fetch-exception.add | 1 + config/description.exs | 6 ++++++ docs/configuration/cheatsheet.md | 1 + lib/pleroma/web/plugs/http_signature_plug.ex | 20 ++++++++++++++----- .../web/plugs/http_signature_plug_test.exs | 19 ++++++++++++++++++ 5 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 changelog.d/auth-fetch-exception.add diff --git a/changelog.d/auth-fetch-exception.add b/changelog.d/auth-fetch-exception.add new file mode 100644 index 0000000000..98efb903eb --- /dev/null +++ b/changelog.d/auth-fetch-exception.add @@ -0,0 +1 @@ +HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions configuration \ No newline at end of file diff --git a/config/description.exs b/config/description.exs index b152981c41..2fed1a152f 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1771,6 +1771,12 @@ type: :boolean, description: "Require HTTP signatures for AP fetches" }, + %{ + key: :authorized_fetch_mode_exceptions, + type: {:list, :string}, + description: + "List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)" + }, %{ key: :note_replies_output_limit, type: :integer, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index a4cae4dbb3..06933ba762 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -279,6 +279,7 @@ Notes: * `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question * `sign_object_fetches`: Sign object fetches with HTTP signatures * `authorized_fetch_mode`: Require HTTP signatures for AP fetches +* `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this) ## Pleroma.User diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index e814efc2cb..7ec2026620 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do + alias Pleroma.Helpers.InetHelper + import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] require Logger @@ -89,12 +91,20 @@ defp has_signature_header?(conn) do defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn - defp maybe_require_signature(conn) do + defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do - conn - |> put_status(:unauthorized) - |> text("Request not signed") - |> halt() + exceptions = + Pleroma.Config.get([:activitypub, :authorized_fetch_mode_exceptions], []) + |> Enum.map(&InetHelper.parse_cidr/1) + + if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do + conn + else + conn + |> put_status(:unauthorized) + |> text("Request not signed") + |> halt() + end else conn end diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 2d8fba3cd2..deb7e4a230 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -81,5 +81,24 @@ test "halts the connection when `signature` header is not present", %{conn: conn assert conn.state == :sent assert conn.resp_body == "Request not signed" end + + test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do + clear_config([:activitypub, :authorized_fetch_mode_exceptions], ["192.168.0.0/24"]) + + with_mock HTTPSignatures, validate_conn: fn _ -> false end do + conn = + conn + |> Map.put(:remote_ip, {192, 168, 0, 1}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.remote_ip == {192, 168, 0, 1} + assert conn.halted == false + assert called(HTTPSignatures.validate_conn(:_)) + end + end end end From 7dfd148ff8a4f2d349d6d6f92d788effdaab36f3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 18:32:26 -0500 Subject: [PATCH 07/61] Logger metadata for inbound federation requests --- config/config.exs | 4 ++-- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 32c8509be3..5375176889 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:request_id] + metadata: [:actor, :request_id, :type] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:request_id] + metadata: [:actor, :request_id, :type] config :mime, :types, %{ "application/xml" => ["xml"], diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e38a949664..d2b2cae0bf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do when action in [:activity, :object] ) + plug(:log_inbox_metadata when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -521,6 +522,13 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end + defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do + Logger.metadata(actor: actor, type: type) + conn + end + + defp log_inbox_metadata(conn, _), do: conn + def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <- ActivityPub.upload( From 40823462e7779fb79a4fcd458daa5e7095a6030b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 17 Dec 2023 18:20:22 -0500 Subject: [PATCH 08/61] Logger metadata for request path and authenticated user --- config/config.exs | 4 ++-- lib/pleroma/web/endpoint.ex | 2 ++ lib/pleroma/web/plugs/logger_metadata_path.ex | 12 ++++++++++++ lib/pleroma/web/plugs/logger_metadata_user.ex | 18 ++++++++++++++++++ lib/pleroma/web/router.ex | 8 ++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/plugs/logger_metadata_path.ex create mode 100644 lib/pleroma/web/plugs/logger_metadata_user.ex diff --git a/config/config.exs b/config/config.exs index 5375176889..83e7a33e3e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:actor, :request_id, :type] + metadata: [:actor, :path, :request_id, :type, :user] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:actor, :request_id, :type] + metadata: [:actor, :path, :request_id, :type, :user] config :mime, :types, %{ "application/xml" => ["xml"], diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 2e2104904e..fef907ace6 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) + plug(Pleroma.Web.Plugs.LoggerMetadataPath) + plug(Pleroma.Web.Plugs.SetLocalePlug) plug(CORSPlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug) diff --git a/lib/pleroma/web/plugs/logger_metadata_path.ex b/lib/pleroma/web/plugs/logger_metadata_path.ex new file mode 100644 index 0000000000..a5553cfc8c --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_path.ex @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataPath do + def init(opts), do: opts + + def call(conn, _) do + Logger.metadata(path: conn.request_path) + conn + end +end diff --git a/lib/pleroma/web/plugs/logger_metadata_user.ex b/lib/pleroma/web/plugs/logger_metadata_user.ex new file mode 100644 index 0000000000..6a5c0041de --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_user.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataUser do + alias Pleroma.User + + def init(opts), do: opts + + def call(%{assigns: %{user: user = %User{}}} = conn, _) do + Logger.metadata(user: user.nickname) + conn + end + + def call(conn, _) do + conn + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4fe0cb02f2..f0414cc35c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do pipeline :browser do plug(:accepts, ["html"]) plug(:fetch_session) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :oauth do @@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do plug(:fetch_session) plug(:authenticate) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :no_auth_or_privacy_expectations_api do plug(:base_api) plug(:after_auth) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported) @@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do pipeline :api do plug(:expect_public_instance_or_user_authentication) plug(:no_auth_or_privacy_expectations_api) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :authenticated_api do plug(:expect_user_authentication) plug(:no_auth_or_privacy_expectations_api) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :admin_api do @@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.UserIsStaffPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :require_admin do @@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do plug(:browser) plug(:authenticate) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :well_known do @@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do pipeline :pleroma_api do plug(:accepts, ["html", "json"]) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :mailbox_preview do From 99cee755d8798c0743b96fb11e55f283f0195b85 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 17 Dec 2023 18:20:34 -0500 Subject: [PATCH 09/61] Show Logger metadata in dev --- config/dev.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index fe8de5045a..f23719fe3d 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -35,8 +35,8 @@ # configured to run both http and https servers on # different ports. -# Do not include metadata nor timestamps in development logs -config :logger, :console, format: "[$level] $message\n" +# Do not include timestamps in development logs +config :logger, :console, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. From 462d5aa5cbe3194bad56a7c973e9b200742ef6ca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Mar 2024 20:53:40 -0400 Subject: [PATCH 10/61] logger: remove request_id metadata which is not useful --- config/config.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 83e7a33e3e..383eb37683 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:actor, :path, :request_id, :type, :user] + metadata: [:actor, :path, :type, :user] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:actor, :path, :request_id, :type, :user] + metadata: [:actor, :path, :type, :user] config :mime, :types, %{ "application/xml" => ["xml"], From cd7e2138d11901fc7a0c8c2f22b7a5d57383a555 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 14:13:37 +0400 Subject: [PATCH 11/61] Search: Basic Qdrant/Ollama search --- config/config.exs | 9 ++ lib/mix/tasks/pleroma/search/indexer.ex | 60 ++++++++++++ lib/pleroma/search/qdrant_search.ex | 117 ++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 lib/mix/tasks/pleroma/search/indexer.ex create mode 100644 lib/pleroma/search/qdrant_search.ex diff --git a/config/config.exs b/config/config.exs index b69044a2be..f74eda6b28 100644 --- a/config/config.exs +++ b/config/config.exs @@ -915,6 +915,15 @@ config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000 +config :pleroma, Pleroma.Search.QdrantSearch, + qdrant_url: "http://127.0.0.1:6333/", + qdrant_api_key: nil, + ollama_url: "http://127.0.0.1:11434", + ollama_model: "snowflake-arctic-embed:xs", + qdrant_index_configuration: %{ + vectors: %{size: 384, distance: "Cosine"} + } + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex new file mode 100644 index 0000000000..ffa2f3c94a --- /dev/null +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Indexer do + import Mix.Pleroma + import Ecto.Query + + alias Pleroma.Workers.SearchIndexingWorker + + def run(["index" | options]) do + {options, [], []} = + OptionParser.parse( + options, + strict: [ + limit: :integer + ] + ) + + start_pleroma() + + limit = Keyword.get(options, :limit, 100_000) + + per_step = 1000 + chunks = max(div(limit, per_step), 1) + + 1..chunks + |> Enum.each(fn step -> + q = + from(a in Pleroma.Activity, + limit: ^per_step, + offset: ^per_step * (^step - 1), + select: [:id], + order_by: [desc: :id] + ) + + {:ok, ids} = + Pleroma.Repo.transaction(fn -> + Pleroma.Repo.stream(q, timeout: :infinity) + |> Enum.map(fn a -> + a.id + end) + end) + + IO.puts("Got #{length(ids)} activities, adding to indexer") + + ids + |> Enum.chunk_every(100) + |> Enum.each(fn chunk -> + IO.puts("Adding #{length(chunk)} activities to indexing queue") + + chunk + |> Enum.map(fn id -> + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) + end) + |> Oban.insert_all() + end) + end) + end +end diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex new file mode 100644 index 0000000000..fcaa9e6864 --- /dev/null +++ b/lib/pleroma/search/qdrant_search.ex @@ -0,0 +1,117 @@ +defmodule Pleroma.Search.QdrantSearch do + @behaviour Pleroma.Search.SearchBackend + import Ecto.Query + alias Pleroma.Activity + + alias __MODULE__.QdrantClient + alias __MODULE__.OllamaClient + + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + + def initialize_index() do + payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) + QdrantClient.put("/collections/posts", payload) + end + + def drop_index() do + QdrantClient.delete("/collections/posts") + end + + def get_embedding(text) do + with {:ok, %{body: %{"embedding" => embedding}}} <- + OllamaClient.post("/api/embeddings", %{ + prompt: text, + model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) + }) + |> IO.inspect() do + {:ok, embedding} + else + _ -> + {:error, "Failed to get embedding"} + end + end + + defp build_index_payload(activity, embedding) do + %{ + points: [ + %{ + id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(), + vector: embedding + } + ] + } + end + + defp build_search_payload(embedding) do + %{ + vector: embedding, + limit: 20 + } + end + + @impl true + def add_to_index(activity) do + # This will only index public or unlisted notes + maybe_search_data = object_to_search_data(activity.object) + IO.puts("TRYING TO INDEX\n\n") + + if activity.data["type"] == "Create" and maybe_search_data do + with {:ok, embedding} <- get_embedding(maybe_search_data.content), + {:ok, %{status: 200}} <- + QdrantClient.put( + "/collections/posts/points", + build_index_payload(activity, embedding) + ) do + :ok + else + e -> {:error, e} + end + else + :ok + end + end + + @impl true + def search(_user, query, _options) do + with {:ok, embedding} <- get_embedding(query), + {:ok, %{body: %{"result" => result}}} <- + QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do + ids = + Enum.map(result, fn %{"id" => id} -> + Ecto.UUID.dump!(id) + end) + + from(a in Activity, where: a.id in ^ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) + |> Pleroma.Repo.all() + else + _ -> + [] + end + end + + @impl true + def remove_from_index(_object) do + :ok + end +end + +defmodule Pleroma.Search.QdrantSearch.OllamaClient do + use Tesla + + plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) + plug(Tesla.Middleware.JSON) +end + +defmodule Pleroma.Search.QdrantSearch.QdrantClient do + use Tesla + + plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])} + ]) +end From bb08a766f4c1bd84c98e245c1871c46fcc7c7a8d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 14:26:41 +0400 Subject: [PATCH 12/61] QdrantSearch: Remove debugging stuff --- lib/pleroma/search/qdrant_search.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index fcaa9e6864..726a30b3b6 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -22,8 +22,7 @@ def get_embedding(text) do OllamaClient.post("/api/embeddings", %{ prompt: text, model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) - }) - |> IO.inspect() do + }) do {:ok, embedding} else _ -> @@ -53,7 +52,6 @@ defp build_search_payload(embedding) do def add_to_index(activity) do # This will only index public or unlisted notes maybe_search_data = object_to_search_data(activity.object) - IO.puts("TRYING TO INDEX\n\n") if activity.data["type"] == "Create" and maybe_search_data do with {:ok, embedding} <- get_embedding(maybe_search_data.content), From 1490ff30af7001adc386b4fec54c62e1a524d7d6 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 15:09:38 +0400 Subject: [PATCH 13/61] QdrantSearch: Add query prefix. --- lib/pleroma/search/qdrant_search.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 726a30b3b6..31e7754ae5 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -71,6 +71,8 @@ def add_to_index(activity) do @impl true def search(_user, query, _options) do + query = "Represent this sentence for searching relevant passages: #{query}" + with {:ok, embedding} <- get_embedding(query), {:ok, %{body: %{"result" => result}}} <- QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do From c50f0f31f418037063bd97efcdc0f60b89594212 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 16:56:58 +0400 Subject: [PATCH 14/61] Docs/Search: Add basic documentation of the qdrant search --- docs/configuration/search.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 0316c9bf43..682d1e52a1 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -10,6 +10,12 @@ To use built-in search that has no external dependencies, set the search module While it has no external dependencies, it has problems with performance and relevancy. +## QdrantSearch + +This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings, for now only the [Ollama](Ollama) api is supported. + +The default settings will support a setup where both Ollama and Qdrant run on the same system as pleroma. The embedding model used by Ollama will need to be pulled first (e.g. `ollama pull snowflake-arctic-embed:xs`) for the embedding to work. + ## Meilisearch Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million From 1261c43a7af48ed6e6753461944659391c4c58cc Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 17:19:36 +0400 Subject: [PATCH 15/61] SearchBackend: Add create_index --- lib/mix/tasks/pleroma/search/indexer.ex | 6 ++++++ lib/pleroma/search/qdrant_search.ex | 3 ++- lib/pleroma/search/search_backend.ex | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index ffa2f3c94a..326646b699 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -8,6 +8,12 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do alias Pleroma.Workers.SearchIndexingWorker + def run(["create_index"]) do + Application.ensure_all_started(:pleroma) + + Pleroma.Config.get([Pleroma.Search, :module]).create_index() + end + def run(["index" | options]) do {options, [], []} = OptionParser.parse( diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 31e7754ae5..315262cb36 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -8,7 +8,8 @@ defmodule Pleroma.Search.QdrantSearch do import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] - def initialize_index() do + @impl true + def create_index() do payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) QdrantClient.put("/collections/posts", payload) end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 68bc48cec2..5be0169d03 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -21,4 +21,9 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Create the index + """ + @callback create_index() :: :ok | {:error, any()} end From a9be4907c0d7b34e5564584d2d040632c32f2aa3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 16 May 2024 10:47:24 +0400 Subject: [PATCH 16/61] SearchBackend: Add drop_index --- lib/mix/tasks/pleroma/search/indexer.ex | 18 ++++++++++++++++-- lib/pleroma/search/database_search.ex | 6 ++++++ lib/pleroma/search/meilisearch.ex | 6 ++++++ lib/pleroma/search/qdrant_search.ex | 14 ++++++++++++-- lib/pleroma/search/search_backend.ex | 5 +++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index 326646b699..81a9fced63 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -9,9 +9,23 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do alias Pleroma.Workers.SearchIndexingWorker def run(["create_index"]) do - Application.ensure_all_started(:pleroma) + start_pleroma() - Pleroma.Config.get([Pleroma.Search, :module]).create_index() + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do + IO.puts("Index created") + else + e -> IO.puts("Could not create index: #{inspect(e)}") + end + end + + def run(["drop_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do + IO.puts("Index dropped") + else + e -> IO.puts("Could not drop index: #{inspect(e)}") + end end def run(["index" | options]) do diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 31bfc7e338..24a1ff4312 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -48,6 +48,12 @@ def add_to_index(_activity), do: :ok @impl true def remove_from_index(_object), do: :ok + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2bff663e88..50f5984d63 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + defp meili_headers do private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 315262cb36..4bd35c17c7 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -11,11 +11,21 @@ defmodule Pleroma.Search.QdrantSearch do @impl true def create_index() do payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) - QdrantClient.put("/collections/posts", payload) + + with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do + :ok + else + e -> {:error, e} + end end + @impl true def drop_index() do - QdrantClient.delete("/collections/posts") + with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do + :ok + else + e -> {:error, e} + end end def get_embedding(text) do diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 5be0169d03..9735ab3f4a 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -26,4 +26,9 @@ defmodule Pleroma.Search.SearchBackend do Create the index """ @callback create_index() :: :ok | {:error, any()} + + @doc """ + Drop the index + """ + @callback drop_index() :: :ok | {:error, any()} end From 069ce4448c556af90293cde9b9872c3d53eb894b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 11:55:17 +0400 Subject: [PATCH 17/61] Add basic fastembed server --- python/fastembed-server.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 python/fastembed-server.py diff --git a/python/fastembed-server.py b/python/fastembed-server.py new file mode 100644 index 0000000000..fa3f7c82b5 --- /dev/null +++ b/python/fastembed-server.py @@ -0,0 +1,21 @@ +from fastembed import TextEmbedding +from fastapi import FastAPI +from pydantic import BaseModel + +model = TextEmbedding("snowflake/snowflake-arctic-embed-xs") + +app = FastAPI() + +class EmbeddingRequest(BaseModel): + model: str + prompt: str + +@app.post("/api/embeddings") +def embeddings(request: EmbeddingRequest): + embeddings = next(model.embed(request.prompt)).tolist() + return {"embedding": embeddings} + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=11345) From 769773a500d4c6ec021776b493f7d98c9f87e81e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 12:08:42 +0400 Subject: [PATCH 18/61] Add dockerfile --- python/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 python/Dockerfile diff --git a/python/Dockerfile b/python/Dockerfile new file mode 100644 index 0000000000..f83c1c1b3d --- /dev/null +++ b/python/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.9 + +WORKDIR /code +COPY fastembed-server.py /workdir/fastembed-server.py + +RUN pip install --no-cache-dir --upgrade fastembed fastapi uvicorn + +CMD ["python", "/workdir/fastembed-server.py"] From 61e9027131843858b017d3b7c18c3a396d5656a9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 12:19:42 +0400 Subject: [PATCH 19/61] Add docker compose file for fastembed server --- python/compose.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 python/compose.yml diff --git a/python/compose.yml b/python/compose.yml new file mode 100644 index 0000000000..d4cb31722f --- /dev/null +++ b/python/compose.yml @@ -0,0 +1,5 @@ +services: + web: + build: . + ports: + - "11345:11345" From 933117785fb1b5b671c61d09671cf6418b105187 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 13:43:47 +0400 Subject: [PATCH 20/61] QdrantSearch: Add basic test --- lib/pleroma/search/qdrant_search.ex | 11 ++-- test/pleroma/search/qdrant_search_test.exs | 65 ++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 test/pleroma/search/qdrant_search_test.exs diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 4bd35c17c7..2d9315a2fc 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -5,12 +5,13 @@ defmodule Pleroma.Search.QdrantSearch do alias __MODULE__.QdrantClient alias __MODULE__.OllamaClient + alias Pleroma.Config.Getting, as: Config import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] @impl true def create_index() do - payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) + payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do :ok @@ -32,7 +33,7 @@ def get_embedding(text) do with {:ok, %{body: %{"embedding" => embedding}}} <- OllamaClient.post("/api/embeddings", %{ prompt: text, - model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) + model: Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) }) do {:ok, embedding} else @@ -111,15 +112,17 @@ def remove_from_index(_object) do defmodule Pleroma.Search.QdrantSearch.OllamaClient do use Tesla + alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) plug(Tesla.Middleware.JSON) end defmodule Pleroma.Search.QdrantSearch.QdrantClient do use Tesla + alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) plug(Tesla.Middleware.JSON) plug(Tesla.Middleware.Headers, [ diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs new file mode 100644 index 0000000000..9be246a9a6 --- /dev/null +++ b/test/pleroma/search/qdrant_search_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.QdrantSearchTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + import Mox + + alias Pleroma.Web.CommonAPI + alias Pleroma.UnstubbedConfigMock, as: Config + alias Pleroma.Search.QdrantSearch + alias Pleroma.Workers.SearchIndexingWorker + + describe "Qdrant search" do + test "indexes a public post on creation" do + user = insert(:user) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://ollama.url/api/embeddings"} -> + send(self(), "posted_to_ollama") + Tesla.Mock.json(%{embedding: [1, 2, 3]}) + + %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} -> + send(self(), "posted_to_qdrant") + + assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body)) + + Tesla.Mock.json("ok") + end) + + Config + |> expect(:get, 4, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + ollama_model: "a_model", + ollama_url: "https://ollama.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued( + worker: SearchIndexingWorker, + args: args + ) + + assert :ok = perform_job(SearchIndexingWorker, args) + assert_received("posted_to_ollama") + assert_received("posted_to_qdrant") + end + end +end From e3933a067feae1f087616f675657d6ff99b2782b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 14:04:32 +0400 Subject: [PATCH 21/61] QdrantSearch: Implement post deletion --- lib/pleroma/search/qdrant_search.ex | 18 +++++++++++++----- test/pleroma/search/qdrant_search_test.exs | 16 ++++++++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 2d9315a2fc..acfaaff52c 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -81,6 +81,19 @@ def add_to_index(activity) do end end + @impl true + def remove_from_index(object) do + activity = Activity.get_by_object_ap_id_with_object(object.data["id"]) + id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!() + + with {:ok, %{status: 200}} <- + QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do + :ok + else + e -> {:error, e} + end + end + @impl true def search(_user, query, _options) do query = "Represent this sentence for searching relevant passages: #{query}" @@ -103,11 +116,6 @@ def search(_user, query, _options) do [] end end - - @impl true - def remove_from_index(_object) do - :ok - end end defmodule Pleroma.Search.QdrantSearch.OllamaClient do diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 9be246a9a6..e816311aa1 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -15,7 +15,7 @@ defmodule Pleroma.Search.QdrantSearchTest do alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do - test "indexes a public post on creation" do + test "indexes a public post on creation, deletes from the index on deletion" do user = insert(:user) Tesla.Mock.mock(fn @@ -29,10 +29,14 @@ test "indexes a public post on creation" do assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body)) Tesla.Mock.json("ok") + + %{method: :post, url: "https://qdrant.url/collections/posts/points/delete"} -> + send(self(), "deleted_from_qdrant") + Tesla.Mock.json("ok") end) Config - |> expect(:get, 4, fn + |> expect(:get, 6, fn [Pleroma.Search, :module], nil -> QdrantSearch @@ -60,6 +64,14 @@ test "indexes a public post on creation" do assert :ok = perform_job(SearchIndexingWorker, args) assert_received("posted_to_ollama") assert_received("posted_to_qdrant") + + {:ok, _} = CommonAPI.delete(activity.id, user) + + delete_args = %{"op" => "remove_from_index", "object" => activity.object.id} + assert_enqueued(worker: SearchIndexingWorker, args: delete_args) + assert :ok = perform_job(SearchIndexingWorker, delete_args) + + assert_received("deleted_from_qdrant") end end end From 39525bcec7c685cb28ca4702b6e145a78e733fee Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 14:07:47 +0400 Subject: [PATCH 22/61] Add qdrant changelog --- changelog.d/qdrant_search.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/qdrant_search.add diff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add new file mode 100644 index 0000000000..6f9e39e237 --- /dev/null +++ b/changelog.d/qdrant_search.add @@ -0,0 +1 @@ +Add Qdrant/Ollama search From 3345ddd2d4ef380929cc231118a5fb6486c0bd5c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 15:02:22 +0400 Subject: [PATCH 23/61] Linting --- lib/pleroma/search/qdrant_search.ex | 11 ++++++----- test/pleroma/search/qdrant_search_test.exs | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index acfaaff52c..a6c6c6a0d2 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -1,16 +1,17 @@ defmodule Pleroma.Search.QdrantSearch do @behaviour Pleroma.Search.SearchBackend import Ecto.Query - alias Pleroma.Activity - alias __MODULE__.QdrantClient - alias __MODULE__.OllamaClient + alias Pleroma.Activity alias Pleroma.Config.Getting, as: Config + alias __MODULE__.OllamaClient + alias __MODULE__.QdrantClient + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] @impl true - def create_index() do + def create_index do payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do @@ -21,7 +22,7 @@ def create_index() do end @impl true - def drop_index() do + def drop_index do with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do :ok else diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index e816311aa1..698894cdbe 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -9,9 +9,9 @@ defmodule Pleroma.Search.QdrantSearchTest do import Pleroma.Factory import Mox - alias Pleroma.Web.CommonAPI - alias Pleroma.UnstubbedConfigMock, as: Config alias Pleroma.Search.QdrantSearch + alias Pleroma.UnstubbedConfigMock, as: Config + alias Pleroma.Web.CommonAPI alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do From 72ec261a69a7dda7ab95667e425824ab7758b636 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:17:46 +0400 Subject: [PATCH 24/61] B QdrantSearch: Switch to OpenAI api --- changelog.d/qdrant_search.add | 2 +- config/config.exs | 7 ++++--- lib/pleroma/search/qdrant_search.ex | 19 ++++++++++++------- test/pleroma/search/qdrant_search_test.exs | 15 +++++++++------ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add index 6f9e39e237..9801131d1a 100644 --- a/changelog.d/qdrant_search.add +++ b/changelog.d/qdrant_search.add @@ -1 +1 @@ -Add Qdrant/Ollama search +Add Qdrant/OpenAI embedding search diff --git a/config/config.exs b/config/config.exs index f74eda6b28..dd0150c665 100644 --- a/config/config.exs +++ b/config/config.exs @@ -917,9 +917,10 @@ config :pleroma, Pleroma.Search.QdrantSearch, qdrant_url: "http://127.0.0.1:6333/", - qdrant_api_key: nil, - ollama_url: "http://127.0.0.1:11434", - ollama_model: "snowflake-arctic-embed:xs", + qdrant_api_key: "", + openai_url: "http://127.0.0.1:11345", + openai_model: "snowflake", + openai_api_key: "", qdrant_index_configuration: %{ vectors: %{size: 384, distance: "Cosine"} } diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index a6c6c6a0d2..5ae04be783 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Search.QdrantSearch do alias Pleroma.Activity alias Pleroma.Config.Getting, as: Config - alias __MODULE__.OllamaClient + alias __MODULE__.OpenAIClient alias __MODULE__.QdrantClient import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] @@ -31,10 +31,10 @@ def drop_index do end def get_embedding(text) do - with {:ok, %{body: %{"embedding" => embedding}}} <- - OllamaClient.post("/api/embeddings", %{ - prompt: text, - model: Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) + with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <- + OpenAIClient.post("/v1/embeddings", %{ + input: text, + model: Config.get([Pleroma.Search.QdrantSearch, :openai_model]) }) do {:ok, embedding} else @@ -119,12 +119,17 @@ def search(_user, query, _options) do end end -defmodule Pleroma.Search.QdrantSearch.OllamaClient do +defmodule Pleroma.Search.QdrantSearch.OpenAIClient do use Tesla alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])) plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"Authorization", + "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"} + ]) end defmodule Pleroma.Search.QdrantSearch.QdrantClient do diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 698894cdbe..a2f9cc7ece 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -19,9 +19,12 @@ test "indexes a public post on creation, deletes from the index on deletion" do user = insert(:user) Tesla.Mock.mock(fn - %{method: :post, url: "https://ollama.url/api/embeddings"} -> - send(self(), "posted_to_ollama") - Tesla.Mock.json(%{embedding: [1, 2, 3]}) + %{method: :post, url: "https://openai.url/v1/embeddings"} -> + send(self(), "posted_to_openai") + + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} -> send(self(), "posted_to_qdrant") @@ -42,8 +45,8 @@ test "indexes a public post on creation, deletes from the index on deletion" do [Pleroma.Search.QdrantSearch, key], nil -> %{ - ollama_model: "a_model", - ollama_url: "https://ollama.url", + openai_model: "a_model", + openai_url: "https://openai.url", qdrant_url: "https://qdrant.url" }[key] end) @@ -62,7 +65,7 @@ test "indexes a public post on creation, deletes from the index on deletion" do ) assert :ok = perform_job(SearchIndexingWorker, args) - assert_received("posted_to_ollama") + assert_received("posted_to_openai") assert_received("posted_to_qdrant") {:ok, _} = CommonAPI.delete(activity.id, user) From b9af017a4cf1025c7d8245fa4f1dbcb678ddd4b9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:33:49 +0400 Subject: [PATCH 25/61] B FastembedServer: Switch to OpenAI api, support changing models --- python/fastembed-server.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/fastembed-server.py b/python/fastembed-server.py index fa3f7c82b5..dd4a7a9c88 100644 --- a/python/fastembed-server.py +++ b/python/fastembed-server.py @@ -2,18 +2,20 @@ from fastembed import TextEmbedding from fastapi import FastAPI from pydantic import BaseModel -model = TextEmbedding("snowflake/snowflake-arctic-embed-xs") +models = {} app = FastAPI() class EmbeddingRequest(BaseModel): model: str - prompt: str + input: str -@app.post("/api/embeddings") +@app.post("/v1/embeddings") def embeddings(request: EmbeddingRequest): - embeddings = next(model.embed(request.prompt)).tolist() - return {"embedding": embeddings} + model = models.get(request.model) or TextEmbedding(request.model) + models[request.model] = model + embeddings = next(model.embed(request.input)).tolist() + return {"data": [{"embedding": embeddings}]} if __name__ == "__main__": import uvicorn From c139a9f38c06ab4485b98b56b9ad4cce4d57be12 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:39:54 +0400 Subject: [PATCH 26/61] B Config: Set default Qdrant embedder to our fastembed-api server --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index dd0150c665..c3b20947d1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -919,7 +919,7 @@ qdrant_url: "http://127.0.0.1:6333/", qdrant_api_key: "", openai_url: "http://127.0.0.1:11345", - openai_model: "snowflake", + openai_model: "snowflake/snowflake-arctic-embed-xs", openai_api_key: "", qdrant_index_configuration: %{ vectors: %{size: 384, distance: "Cosine"} From e142ea400a9ed3595f8d432edd90ea26fc7d2eb5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:42:08 +0400 Subject: [PATCH 27/61] Docs: Switch docs from Ollama to OpenAI. --- docs/configuration/search.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 682d1e52a1..388f5acd14 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -12,9 +12,9 @@ While it has no external dependencies, it has problems with performance and rele ## QdrantSearch -This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings, for now only the [Ollama](Ollama) api is supported. +This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`. -The default settings will support a setup where both Ollama and Qdrant run on the same system as pleroma. The embedding model used by Ollama will need to be pulled first (e.g. `ollama pull snowflake-arctic-embed:xs`) for the embedding to work. +The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. ## Meilisearch From dd48810186e3b4ee14e1d3727f37bd470d0711a4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:47:08 +0400 Subject: [PATCH 28/61] B FastembedAPI: Move to more appropriate folder --- {python => supplemental/search/fastembed-api}/Dockerfile | 0 {python => supplemental/search/fastembed-api}/compose.yml | 0 {python => supplemental/search/fastembed-api}/fastembed-server.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {python => supplemental/search/fastembed-api}/Dockerfile (100%) rename {python => supplemental/search/fastembed-api}/compose.yml (100%) rename {python => supplemental/search/fastembed-api}/fastembed-server.py (100%) diff --git a/python/Dockerfile b/supplemental/search/fastembed-api/Dockerfile similarity index 100% rename from python/Dockerfile rename to supplemental/search/fastembed-api/Dockerfile diff --git a/python/compose.yml b/supplemental/search/fastembed-api/compose.yml similarity index 100% rename from python/compose.yml rename to supplemental/search/fastembed-api/compose.yml diff --git a/python/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py similarity index 100% rename from python/fastembed-server.py rename to supplemental/search/fastembed-api/fastembed-server.py From 8329ad521419119f89e3e2577269475190cfe921 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:59:03 +0400 Subject: [PATCH 29/61] B FastembedAPI: Add requirements.txt --- supplemental/search/fastembed-api/Dockerfile | 3 ++- supplemental/search/fastembed-api/requirements.txt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 supplemental/search/fastembed-api/requirements.txt diff --git a/supplemental/search/fastembed-api/Dockerfile b/supplemental/search/fastembed-api/Dockerfile index f83c1c1b3d..c1e0ef51f3 100644 --- a/supplemental/search/fastembed-api/Dockerfile +++ b/supplemental/search/fastembed-api/Dockerfile @@ -2,7 +2,8 @@ FROM python:3.9 WORKDIR /code COPY fastembed-server.py /workdir/fastembed-server.py +COPY requirements.txt /workdir/requirements.txt -RUN pip install --no-cache-dir --upgrade fastembed fastapi uvicorn +RUN pip install -r /workdir/requirements.txt CMD ["python", "/workdir/fastembed-server.py"] diff --git a/supplemental/search/fastembed-api/requirements.txt b/supplemental/search/fastembed-api/requirements.txt new file mode 100644 index 0000000000..db67a84022 --- /dev/null +++ b/supplemental/search/fastembed-api/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.111.0 +fastembed==0.2.7 +pydantic==1.10.15 +uvicorn==0.29.0 From 23881842ae33a294e344cef0cc2f1385ea6819f9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:04:27 +0400 Subject: [PATCH 30/61] B FastembedAPI: Add readme --- supplemental/search/fastembed-api/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 supplemental/search/fastembed-api/README.md diff --git a/supplemental/search/fastembed-api/README.md b/supplemental/search/fastembed-api/README.md new file mode 100644 index 0000000000..63a037207d --- /dev/null +++ b/supplemental/search/fastembed-api/README.md @@ -0,0 +1,6 @@ +# About +This is a minimal implementation of the [OpenAI Embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings) meant to be used with the QdrantSearch backend. + +# Usage + +The easiest way to run it is to just use docker compose with `docker compose up`. This starts the server on the default configured port. Different models can be used, for a full list of supported models, check the [fastembed documentation](https://qdrant.github.io/fastembed/examples/Supported_Models/). The first time a model is requested it will be downloaded, which can take a few seconds. From 6a3a0cc0f5995185428c92f3c53e9c8524ea6856 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:20:37 +0400 Subject: [PATCH 31/61] Docs: Write docs for the QdrantSearch --- docs/configuration/search.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 388f5acd14..6598e533f6 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -14,7 +14,25 @@ While it has no external dependencies, it has problems with performance and rele This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`. -The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. +The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`: + +https://qdrant.github.io/fastembed/examples/Supported_Models/ + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch + +You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities. + +### Indexing and model options + +To see the available configuration options, check out the QdrantSearch section in `config/config.exs`. + +The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). + +Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`. + +E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024. + +When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other. ## Meilisearch From 6ec306d0684f3c5c05d768a3c431008925f21f15 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:24:24 +0400 Subject: [PATCH 32/61] Docs: Add more information about index memory consumption. --- docs/configuration/search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 6598e533f6..ed85acd2a9 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -26,7 +26,7 @@ You will also need to create the Qdrant index once by running `mix pleroma.searc To see the available configuration options, check out the QdrantSearch section in `config/config.exs`. -The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). +The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail. Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`. From dbaab6f54e306e5fb930ce1ed0699631c8aeaae1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:38:31 +0400 Subject: [PATCH 33/61] Docs: Mention running the Qdrant server --- docs/configuration/search.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index ed85acd2a9..d34f84d4f9 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -16,10 +16,10 @@ This uses the vector search engine [Qdrant](https://qdrant.tech) to search the p The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`: -https://qdrant.github.io/fastembed/examples/Supported_Models/ - > config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch +Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions. + You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities. ### Indexing and model options From 1b4f1db9b2990f725a06f0dff41980c51853c5e9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 14:41:05 +0400 Subject: [PATCH 34/61] QdrantSearch: Support pagination. --- lib/pleroma/search/qdrant_search.ex | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 5ae04be783..283c43075e 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -54,10 +54,11 @@ defp build_index_payload(activity, embedding) do } end - defp build_search_payload(embedding) do + defp build_search_payload(embedding, options) do %{ vector: embedding, - limit: 20 + limit: options[:limit] || 20, + offset: options[:offset] || 0 } end @@ -96,12 +97,15 @@ def remove_from_index(object) do end @impl true - def search(_user, query, _options) do + def search(_user, query, options) do query = "Represent this sentence for searching relevant passages: #{query}" with {:ok, embedding} <- get_embedding(query), {:ok, %{body: %{"result" => result}}} <- - QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do + QdrantClient.post( + "/collections/posts/points/search", + build_search_payload(embedding, options) + ) do ids = Enum.map(result, fn %{"id" => id} -> Ecto.UUID.dump!(id) From 94e4f215896dc7976a54fd146daf3e286602925a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 23 May 2024 14:38:30 +0400 Subject: [PATCH 35/61] QdrantSearch: Deal with actor restrictions --- lib/pleroma/search/qdrant_search.ex | 22 ++++- test/pleroma/search/qdrant_search_test.exs | 95 +++++++++++++++++++++- 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 283c43075e..9cb34ef71f 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -43,23 +43,41 @@ def get_embedding(text) do end end + defp actor_from_activity(%{data: %{"actor" => actor}}) do + actor + end + + defp actor_from_activity(_), do: nil + defp build_index_payload(activity, embedding) do + actor = actor_from_activity(activity) + published_at = activity.data["published"] + %{ points: [ %{ id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(), - vector: embedding + vector: embedding, + payload: %{actor: actor, published_at: published_at} } ] } end defp build_search_payload(embedding, options) do - %{ + base = %{ vector: embedding, limit: options[:limit] || 20, offset: options[:offset] || 0 } + + if options[:actor] do + Map.put(base, :filter, %{ + must: [%{key: "actor", match: %{value: options[:actor].ap_id}}] + }) + else + base + end end @impl true diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index a2f9cc7ece..371074dcfb 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -15,6 +15,94 @@ defmodule Pleroma.Search.QdrantSearchTest do alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do + test "searches for a term by encoding it and sending it to qdrant" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + openai_model: "a_model", + openai_url: "https://openai.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + Tesla.Mock.mock(fn + %{url: "https://openai.url/v1/embeddings", method: :post} -> + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) + + %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} -> + data = Jason.decode!(body) + refute data["filter"] + + Tesla.Mock.json(%{ + result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}] + }) + end) + + results = QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{}) + + assert results == [activity] + end + + test "for a given actor, ask for only relevant matches" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + openai_model: "a_model", + openai_url: "https://openai.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + Tesla.Mock.mock(fn + %{url: "https://openai.url/v1/embeddings", method: :post} -> + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) + + %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} -> + data = Jason.decode!(body) + + assert data["filter"] == %{ + "must" => [%{"key" => "actor", "match" => %{"value" => user.ap_id}}] + } + + Tesla.Mock.json(%{ + result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}] + }) + end) + + results = + QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{actor: user}) + + assert results == [activity] + end + test "indexes a public post on creation, deletes from the index on deletion" do user = insert(:user) @@ -29,7 +117,12 @@ test "indexes a public post on creation, deletes from the index on deletion" do %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} -> send(self(), "posted_to_qdrant") - assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body)) + data = Jason.decode!(body) + %{"points" => [%{"vector" => vector, "payload" => payload}]} = data + + assert vector == [1, 2, 3] + assert payload["actor"] + assert payload["published_at"] Tesla.Mock.json("ok") From a566ad56e1434715d00067b1e49be66b6787f5ba Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 23 May 2024 18:55:16 +0400 Subject: [PATCH 36/61] QdrantSearch: Fix actor / author restriction --- lib/pleroma/search/qdrant_search.ex | 4 ++-- test/pleroma/search/qdrant_search_test.exs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 9cb34ef71f..19e8cd4bf2 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -71,9 +71,9 @@ defp build_search_payload(embedding, options) do offset: options[:offset] || 0 } - if options[:actor] do + if author = options[:author] do Map.put(base, :filter, %{ - must: [%{key: "actor", match: %{value: options[:actor].ap_id}}] + must: [%{key: "actor", match: %{value: author.ap_id}}] }) else base diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 371074dcfb..46485392e8 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -98,7 +98,7 @@ test "for a given actor, ask for only relevant matches" do end) results = - QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{actor: user}) + QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{author: user}) assert results == [activity] end From 8b76f56050a609bf562053cb7201a9204901490e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 13:57:42 +0400 Subject: [PATCH 37/61] QdrantSearch: Add healthcheck for qdrant --- lib/pleroma/search/qdrant_search.ex | 11 +++++++++++ test/pleroma/search/qdrant_search_test.exs | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 19e8cd4bf2..3c3ffce16b 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -139,6 +139,17 @@ def search(_user, query, options) do [] end end + + @impl true + def healthcheck_endpoints do + qdrant_health = + Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]) + |> URI.parse() + |> Map.put(:path, "/healthz") + |> URI.to_string() + + [qdrant_health] + end end defmodule Pleroma.Search.QdrantSearch.OpenAIClient do diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 46485392e8..b389aa8165 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -15,6 +15,18 @@ defmodule Pleroma.Search.QdrantSearchTest do alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do + test "returns the correct healthcheck endpoints" do + Config + |> expect(:get, 1, fn + [Pleroma.Search.QdrantSearch, key], nil -> + %{qdrant_url: "https://qdrant.url"}[key] + end) + + health_endpoints = QdrantSearch.healthcheck_endpoints() + + assert "https://qdrant.url/healthz" in health_endpoints + end + test "searches for a term by encoding it and sending it to qdrant" do user = insert(:user) From ec3f3fef7798111641f08020d5fd7ae16e407b89 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 14:15:04 +0400 Subject: [PATCH 38/61] Fastembed Server: Add health check endpoint --- supplemental/search/fastembed-api/fastembed-server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/supplemental/search/fastembed-api/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py index dd4a7a9c88..02da69db26 100644 --- a/supplemental/search/fastembed-api/fastembed-server.py +++ b/supplemental/search/fastembed-api/fastembed-server.py @@ -17,6 +17,10 @@ def embeddings(request: EmbeddingRequest): embeddings = next(model.embed(request.input)).tolist() return {"data": [{"embedding": embeddings}]} +@app.get("/health") +def health(): + return {"status": "ok"} + if __name__ == "__main__": import uvicorn From f4c04e6b2dce6d75d148ca520aaef27005ecaa82 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 14:21:55 +0400 Subject: [PATCH 39/61] QdrantSearch: Add health checks. --- config/config.exs | 3 +++ lib/pleroma/search/qdrant_search.ex | 4 +++- test/pleroma/search/qdrant_search_test.exs | 20 +++++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index d891a52186..f388dfe523 100644 --- a/config/config.exs +++ b/config/config.exs @@ -919,6 +919,9 @@ qdrant_url: "http://127.0.0.1:6333/", qdrant_api_key: "", openai_url: "http://127.0.0.1:11345", + # The healthcheck url has to be set to nil when used with the real openai + # API, as it doesn't have a healthcheck endpoint. + openai_healthcheck_url: "http://127.0.0.1:11345/health", openai_model: "snowflake/snowflake-arctic-embed-xs", openai_api_key: "", qdrant_index_configuration: %{ diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 3c3ffce16b..429ae05b8f 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -148,7 +148,9 @@ def healthcheck_endpoints do |> Map.put(:path, "/healthz") |> URI.to_string() - [qdrant_health] + openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url]) + + [qdrant_health, openai_health] |> Enum.filter(& &1) end end diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index b389aa8165..47a77a3912 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -16,15 +16,29 @@ defmodule Pleroma.Search.QdrantSearchTest do describe "Qdrant search" do test "returns the correct healthcheck endpoints" do + # No openai healthcheck URL Config - |> expect(:get, 1, fn + |> expect(:get, 2, fn [Pleroma.Search.QdrantSearch, key], nil -> %{qdrant_url: "https://qdrant.url"}[key] end) - health_endpoints = QdrantSearch.healthcheck_endpoints() + [health_endpoint] = QdrantSearch.healthcheck_endpoints() - assert "https://qdrant.url/healthz" in health_endpoints + assert "https://qdrant.url/healthz" == health_endpoint + + # Set openai healthcheck URL + Config + |> expect(:get, 2, fn + [Pleroma.Search.QdrantSearch, key], nil -> + %{qdrant_url: "https://qdrant.url", openai_healthcheck_url: "https://openai.url/health"}[ + key + ] + end) + + [_, health_endpoint] = QdrantSearch.healthcheck_endpoints() + + assert "https://openai.url/health" == health_endpoint end test "searches for a term by encoding it and sending it to qdrant" do From ddf103eca04c9571ba8310915556cc51cd4a9af8 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 14:35:08 +0400 Subject: [PATCH 40/61] QdrantSearch: Fetch a post in search if possible. --- lib/pleroma/search/qdrant_search.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 429ae05b8f..b659bb682c 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Search.QdrantSearch do alias __MODULE__.QdrantClient import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3] @impl true def create_index do @@ -115,8 +116,8 @@ def remove_from_index(object) do end @impl true - def search(_user, query, options) do - query = "Represent this sentence for searching relevant passages: #{query}" + def search(user, original_query, options) do + query = "Represent this sentence for searching relevant passages: #{original_query}" with {:ok, embedding} <- get_embedding(query), {:ok, %{body: %{"result" => result}}} <- @@ -134,6 +135,7 @@ def search(_user, query, options) do |> Activity.restrict_deactivated_users() |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) |> Pleroma.Repo.all() + |> maybe_fetch(user, original_query) else _ -> [] From a50c657427a2dfe9d48c25529a179fe634d30e48 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:17:02 -0400 Subject: [PATCH 41/61] Add a dedicated connection pool for Rich Media Sharing this pool with regular Media is problematic as Rich Media will connect to many different domains and thrash the pool, but regular Media will have predictable connections to the webservers hosting media for the fediverse servers you peer with. --- config/config.exs | 9 +++++++++ lib/pleroma/web/rich_media/helpers.ex | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 8b9a588b7b..b8030651fa 100644 --- a/config/config.exs +++ b/config/config.exs @@ -827,6 +827,11 @@ max_waiting: 20, recv_timeout: 15_000 ], + rich_media: [ + size: 25, + max_waiting: 20, + recv_timeout: 15_000 + ], upload: [ size: 25, max_waiting: 5, @@ -847,6 +852,10 @@ max_connections: 50, timeout: 150_000 ], + rich_media: [ + max_connections: 50, + timeout: 150_000 + ], upload: [ max_connections: 25, timeout: 300_000 diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 1199944588..ea41bd285a 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -58,7 +58,7 @@ defp check_content_length(headers) do defp http_options do [ - pool: :media, + pool: :rich_media, max_body: Config.get([:rich_media, :max_body], 5_000_000) ] end From 6708f154a4f7ad46b4637d4d566b8cf81e3ebb7b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:18:58 -0400 Subject: [PATCH 42/61] Rework Gun connection pool sizes to make better use of the default 250 connections --- config/config.exs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index b8030651fa..a025defeb3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -818,27 +818,27 @@ config :pleroma, :pools, federation: [ - size: 50, - max_waiting: 10, + size: 75, + max_waiting: 20, recv_timeout: 10_000 ], media: [ - size: 50, + size: 75, max_waiting: 20, recv_timeout: 15_000 ], rich_media: [ size: 25, max_waiting: 20, - recv_timeout: 15_000 - ], + recv_timeout: 15_000 + ], upload: [ size: 25, - max_waiting: 5, + max_waiting: 20, recv_timeout: 15_000 ], default: [ - size: 10, + size: 50, max_waiting: 2, recv_timeout: 5_000 ] From d272eb62cd4da45e5ef82fcc0126d2cf799d292a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:22:45 -0400 Subject: [PATCH 43/61] Trust the connection pools to enforce the concurrency limitations --- .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index c95d35bb91..f10dc3ce5c 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -30,9 +30,7 @@ defp prefetch(url) do if Pleroma.Config.get(:env) == :test do fetch(prefetch_url) else - ConcurrentLimiter.limit(__MODULE__, fn -> - Task.start(fn -> fetch(prefetch_url) end) - end) + Task.start(fn -> fetch(prefetch_url) end) end end end From 37d79b76bb770cb294c0a54777b435dc49c042ab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:24:54 -0400 Subject: [PATCH 44/61] Use the configured http client options for mediaproxy --- .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 10 ++++------ lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index f10dc3ce5c..e5eb6896a1 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @adapter_options [ - pool: :media, - recv_timeout: 10_000 - ] - @impl true def history_awareness, do: :auto @@ -35,7 +30,10 @@ defp prefetch(url) do end end - defp fetch(url), do: HTTP.get(url, [], @adapter_options) + defp fetch(url) do + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) + HTTP.get(url, [], http_client_opts) + end defp preload(%{"object" => %{"attachment" => attachments}} = _message) do Enum.each(attachments, fn diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index c11484ecb5..0b446e0a60 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -54,9 +54,10 @@ def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do defp handle_preview(conn, url) do media_proxy_url = MediaProxy.url(url) + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do + Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do content_type = Tesla.get_header(head_response, "content-type") content_length = Tesla.get_header(head_response, "content-length") content_length = content_length && String.to_integer(content_length) From 8b61d4e3e124c4fafeaabe58a8c7673f767b2871 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:28:31 -0400 Subject: [PATCH 45/61] Changelogs --- changelog.d/mediaproxy-http.fix | 1 + changelog.d/pools.change | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/mediaproxy-http.fix create mode 100644 changelog.d/pools.change diff --git a/changelog.d/mediaproxy-http.fix b/changelog.d/mediaproxy-http.fix new file mode 100644 index 0000000000..4ff6430e08 --- /dev/null +++ b/changelog.d/mediaproxy-http.fix @@ -0,0 +1 @@ +Ensure MediaProxy HTTP requests obey all the defined connection settings diff --git a/changelog.d/pools.change b/changelog.d/pools.change new file mode 100644 index 0000000000..3c689195a4 --- /dev/null +++ b/changelog.d/pools.change @@ -0,0 +1 @@ +HTTP connection pool adjustments From e4f1325f78e9be9fb200358d73794f15794c39bd Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 19:44:41 +0400 Subject: [PATCH 46/61] InetHelper: Don't use deprecated function. --- lib/pleroma/helpers/inet_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex index 3500fc6791..00e18649ec 100644 --- a/lib/pleroma/helpers/inet_helper.ex +++ b/lib/pleroma/helpers/inet_helper.ex @@ -25,6 +25,6 @@ def parse_cidr(proxy) when is_binary(proxy) do InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" end - InetCidr.parse(proxy, true) + InetCidr.parse_cidr!(proxy, true) end end From 284cd0abe5fd34d0bb31281614a7dc9249731b40 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 20:04:12 +0400 Subject: [PATCH 47/61] Add changelog --- changelog.d/support-honk-image-summaries.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/support-honk-image-summaries.add diff --git a/changelog.d/support-honk-image-summaries.add b/changelog.d/support-honk-image-summaries.add new file mode 100644 index 0000000000..052c03f95a --- /dev/null +++ b/changelog.d/support-honk-image-summaries.add @@ -0,0 +1 @@ +Support honk-style attachment summaries as alt-text. From 1c699144d23aa4a86ff8b6ebef7d760ce9e3a4e2 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 21:26:40 +0400 Subject: [PATCH 48/61] HttpSecurityPlug: Don't allow unsafe-eval by default --- config/config.exs | 3 +- config/test.exs | 1 + lib/pleroma/application.ex | 7 +- lib/pleroma/web/plugs/http_security_plug.ex | 49 +++-- .../web/plugs/http_security_plug_test.exs | 208 ++++++++++++++---- 5 files changed, 204 insertions(+), 64 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4752bbbdec..f861daf043 100644 --- a/config/config.exs +++ b/config/config.exs @@ -519,7 +519,8 @@ sts: false, sts_max_age: 31_536_000, ct_max_age: 2_592_000, - referrer_policy: "same-origin" + referrer_policy: "same-origin", + allow_unsafe_eval: false config :cors_plug, max_age: 86_400, diff --git a/config/test.exs b/config/test.exs index 3345bb3a97..b5c9c6e4a0 100644 --- a/config/test.exs +++ b/config/test.exs @@ -154,6 +154,7 @@ config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.UnstubbedConfigMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d266d18364..0d9757b443 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Application do @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] + @compile_env Mix.env() def name, do: @name def version, do: @version @@ -51,7 +52,11 @@ def start(_type, _args) do Pleroma.HTML.compile_scrubbers() Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() - Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + + if @compile_env != :test do + Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + end + Pleroma.ApplicationRequirements.verify!() load_custom_modules() Pleroma.Docs.JSON.compile() diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a27dcd0abb..a1dc6c02ae 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -3,26 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do - alias Pleroma.Config import Plug.Conn require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + def init(opts), do: opts def call(conn, _options) do - if Config.get([:http_security, :enabled]) do + if @config_impl.get([:http_security, :enabled]) do conn |> merge_resp_headers(headers()) - |> maybe_send_sts_header(Config.get([:http_security, :sts])) + |> maybe_send_sts_header(@config_impl.get([:http_security, :sts])) else conn end end def primary_frontend do - with %{"name" => frontend} <- Config.get([:frontends, :primary]), - available <- Config.get([:frontends, :available]), + with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]), + available <- @config_impl.get([:frontends, :available]), %{} = primary_frontend <- Map.get(available, frontend) do {:ok, primary_frontend} end @@ -37,8 +38,8 @@ def custom_http_frontend_headers do end def headers do - referrer_policy = Config.get([:http_security, :referrer_policy]) - report_uri = Config.get([:http_security, :report_uri]) + referrer_policy = @config_impl.get([:http_security, :referrer_policy]) + report_uri = @config_impl.get([:http_security, :report_uri]) custom_http_frontend_headers = custom_http_frontend_headers() headers = [ @@ -86,10 +87,10 @@ def headers do @csp_start [Enum.join(static_csp_rules, ";") <> ";"] defp csp_string do - scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] + scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() - report_uri = Config.get([:http_security, :report_uri]) + report_uri = @config_impl.get([:http_security, :report_uri]) img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" @@ -97,8 +98,8 @@ defp csp_string do # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = - if Config.get([:media_proxy, :enabled]) && - !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do + if @config_impl.get([:media_proxy, :enabled]) && + !@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() { @@ -115,17 +116,21 @@ defp csp_string do end connect_src = - if Config.get(:env) == :dev do + if @config_impl.get(:env) == :dev do [connect_src, " http://localhost:3035/"] else connect_src end script_src = - if Config.get(:env) == :dev do - "script-src 'self' 'unsafe-eval'" + if @config_impl.get([:http_security, :allow_unsafe_eval]) do + if @config_impl.get(:env) == :dev do + "script-src 'self' 'unsafe-eval'" + else + "script-src 'self' 'wasm-unsafe-eval'" + end else - "script-src 'self' 'wasm-unsafe-eval'" + "script-src 'self'" end report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] @@ -161,11 +166,11 @@ defp build_csp_param_from_whitelist(url), do: url defp build_csp_multimedia_source_list do media_proxy_whitelist = [:media_proxy, :whitelist] - |> Config.get() + |> @config_impl.get() |> build_csp_from_whitelist([]) - captcha_method = Config.get([Pleroma.Captcha, :method]) - captcha_endpoint = Config.get([captcha_method, :endpoint]) + captcha_method = @config_impl.get([Pleroma.Captcha, :method]) + captcha_endpoint = @config_impl.get([captcha_method, :endpoint]) base_endpoints = [ @@ -173,7 +178,7 @@ defp build_csp_multimedia_source_list do [Pleroma.Upload, :base_url], [Pleroma.Uploaders.S3, :public_endpoint] ] - |> Enum.map(&Config.get/1) + |> Enum.map(&@config_impl.get/1) [captcha_endpoint | base_endpoints] |> Enum.map(&build_csp_param/1) @@ -200,7 +205,7 @@ defp build_csp_param(url) when is_binary(url) do end def warn_if_disabled do - unless Config.get([:http_security, :enabled]) do + unless Pleroma.Config.get([:http_security, :enabled]) do Logger.warning(" .i;;;;i. iYcviii;vXY: @@ -245,8 +250,8 @@ def warn_if_disabled do end defp maybe_send_sts_header(conn, true) do - max_age_sts = Config.get([:http_security, :sts_max_age]) - max_age_ct = Config.get([:http_security, :ct_max_age]) + max_age_sts = @config_impl.get([:http_security, :sts_max_age]) + max_age_ct = @config_impl.get([:http_security, :ct_max_age]) merge_resp_headers(conn, [ {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"}, diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index c79170382c..80ad1fa7d2 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -3,14 +3,52 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do - use Pleroma.Web.ConnCase + use Pleroma.Web.ConnCase, async: true alias Plug.Conn - describe "http security enabled" do - setup do: clear_config([:http_security, :enabled], true) + import Mox - test "it sends CSP headers when enabled", %{conn: conn} do + setup do + base_config = Pleroma.Config.get([:http_security]) + %{base_config: base_config} + end + + defp mock_config(config, additional \\ %{}) do + Pleroma.UnstubbedConfigMock + |> stub(:get, fn + [:http_security, key] -> config[key] + key -> additional[key] + end) + end + + describe "http security enabled" do + setup %{base_config: base_config} do + %{base_config: Keyword.put(base_config, :enabled, true)} + end + + test "it does not contain unsafe-eval", %{conn: conn, base_config: base_config} do + mock_config(base_config) + + conn = get(conn, "/api/v1/instance") + [header] = Conn.get_resp_header(conn, "content-security-policy") + refute header =~ ~r/unsafe-eval/ + end + + test "with allow_unsafe_eval set, it does contain it", %{conn: conn, base_config: base_config} do + base_config = + base_config + |> Keyword.put(:allow_unsafe_eval, true) + + mock_config(base_config) + + conn = get(conn, "/api/v1/instance") + [header] = Conn.get_resp_header(conn, "content-security-policy") + assert header =~ ~r/unsafe-eval/ + end + + test "it sends CSP headers when enabled", %{conn: conn, base_config: base_config} do + mock_config(base_config) conn = get(conn, "/api/v1/instance") refute Conn.get_resp_header(conn, "x-xss-protection") == [] @@ -22,8 +60,10 @@ test "it sends CSP headers when enabled", %{conn: conn} do refute Conn.get_resp_header(conn, "content-security-policy") == [] end - test "it sends STS headers when enabled", %{conn: conn} do - clear_config([:http_security, :sts], true) + test "it sends STS headers when enabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:sts, true) + |> mock_config() conn = get(conn, "/api/v1/instance") @@ -31,8 +71,10 @@ test "it sends STS headers when enabled", %{conn: conn} do refute Conn.get_resp_header(conn, "expect-ct") == [] end - test "it does not send STS headers when disabled", %{conn: conn} do - clear_config([:http_security, :sts], false) + test "it does not send STS headers when disabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:sts, false) + |> mock_config() conn = get(conn, "/api/v1/instance") @@ -40,19 +82,30 @@ test "it does not send STS headers when disabled", %{conn: conn} do assert Conn.get_resp_header(conn, "expect-ct") == [] end - test "referrer-policy header reflects configured value", %{conn: conn} do - resp = get(conn, "/api/v1/instance") + test "referrer-policy header reflects configured value", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) + resp = get(conn, "/api/v1/instance") assert Conn.get_resp_header(resp, "referrer-policy") == ["same-origin"] - clear_config([:http_security, :referrer_policy], "no-referrer") + base_config + |> Keyword.put(:referrer_policy, "no-referrer") + |> mock_config resp = get(conn, "/api/v1/instance") assert Conn.get_resp_header(resp, "referrer-policy") == ["no-referrer"] end - test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} do + test "it sends `report-to` & `report-uri` CSP response headers", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) + conn = get(conn, "/api/v1/instance") [csp] = Conn.get_resp_header(conn, "content-security-policy") @@ -65,7 +118,11 @@ test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} d "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}" end - test "default values for img-src and media-src with disabled media proxy", %{conn: conn} do + test "default values for img-src and media-src with disabled media proxy", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) conn = get(conn, "/api/v1/instance") [csp] = Conn.get_resp_header(conn, "content-security-policy") @@ -73,60 +130,129 @@ test "default values for img-src and media-src with disabled media proxy", %{con assert csp =~ "img-src 'self' data: blob: https:;" end - test "it sets the Service-Worker-Allowed header", %{conn: conn} do - clear_config([:http_security, :enabled], true) - clear_config([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"}) + test "it sets the Service-Worker-Allowed header", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:enabled, true) - clear_config([:frontends, :available], %{ - "fedi-fe" => %{ - "name" => "fedi-fe", - "custom-http-headers" => [{"service-worker-allowed", "/"}] - } - }) + additional_config = + %{} + |> Map.put([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"}) + |> Map.put( + [:frontends, :available], + %{ + "fedi-fe" => %{ + "name" => "fedi-fe", + "custom-http-headers" => [{"service-worker-allowed", "/"}] + } + } + ) + mock_config(base_config, additional_config) conn = get(conn, "/api/v1/instance") assert Conn.get_resp_header(conn, "service-worker-allowed") == ["/"] end end describe "img-src and media-src" do - setup do - clear_config([:http_security, :enabled], true) - clear_config([:media_proxy, :enabled], true) - clear_config([:media_proxy, :proxy_opts, :redirect_on_failure], false) + setup %{base_config: base_config} do + base_config = + base_config + |> Keyword.put(:enabled, true) + + additional_config = + %{} + |> Map.put([:media_proxy, :enabled], true) + |> Map.put([:media_proxy, :proxy_opts, :redirect_on_failure], false) + |> Map.put([:media_proxy, :whitelist], []) + + %{base_config: base_config, additional_config: additional_config} end - test "media_proxy with base_url", %{conn: conn} do + test "media_proxy with base_url", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example.com" - clear_config([:media_proxy, :base_url], url) + + additional_config = + additional_config + |> Map.put([:media_proxy, :base_url], url) + + mock_config(base_config, additional_config) + assert_media_img_src(conn, url) end - test "upload with base url", %{conn: conn} do + test "upload with base url", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example2.com" - clear_config([Pleroma.Upload, :base_url], url) + + additional_config = + additional_config + |> Map.put([Pleroma.Upload, :base_url], url) + + mock_config(base_config, additional_config) + assert_media_img_src(conn, url) end - test "with S3 public endpoint", %{conn: conn} do + test "with S3 public endpoint", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example3.com" - clear_config([Pleroma.Uploaders.S3, :public_endpoint], url) + + additional_config = + additional_config + |> Map.put([Pleroma.Uploaders.S3, :public_endpoint], url) + + mock_config(base_config, additional_config) assert_media_img_src(conn, url) end - test "with captcha endpoint", %{conn: conn} do - clear_config([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com") + test "with captcha endpoint", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com") + |> Map.put([Pleroma.Captcha, :method], Pleroma.Captcha.Mock) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "https://captcha.com") end - test "with media_proxy whitelist", %{conn: conn} do - clear_config([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"]) + test "with media_proxy whitelist", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"]) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "https://example7.com https://example6.com") end # TODO: delete after removing support bare domains for media proxy whitelist - test "with media_proxy bare domains whitelist (deprecated)", %{conn: conn} do - clear_config([:media_proxy, :whitelist], ["example4.com", "example5.com"]) + test "with media_proxy bare domains whitelist (deprecated)", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([:media_proxy, :whitelist], ["example4.com", "example5.com"]) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "example5.com example4.com") end end @@ -138,8 +264,10 @@ defp assert_media_img_src(conn, url) do assert csp =~ "img-src 'self' data: blob: #{url};" end - test "it does not send CSP headers when disabled", %{conn: conn} do - clear_config([:http_security, :enabled], false) + test "it does not send CSP headers when disabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:enabled, false) + |> mock_config conn = get(conn, "/api/v1/instance") From fc7ce339edc40cb791d321a20f01f2568337b845 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 21:28:20 +0400 Subject: [PATCH 49/61] Cheatsheet: Add allow_unsafe_eval --- docs/configuration/cheatsheet.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index ca2ce63696..78997c4db5 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -472,6 +472,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent. * ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`. * ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. +* `allow_unsafe_eval`: Adds `wasm-unsafe-eval` to the CSP header. Needed for some non-essential frontend features like Flash emulation. ### Pleroma.Web.Plugs.RemoteIp From c67b41415b369d67c25356205bf69de2d99a291c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 11 Jun 2023 20:24:18 +0400 Subject: [PATCH 50/61] Changelog: Add changelog entry. --- changelog.d/3904.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3904.security diff --git a/changelog.d/3904.security b/changelog.d/3904.security new file mode 100644 index 0000000000..04836d4e8c --- /dev/null +++ b/changelog.d/3904.security @@ -0,0 +1 @@ +HTTP Security: By default, don't allow unsafe-eval. The setting needs to be changed to allow Flash emulation. From 0847d9ebafa38007aeef0a6677588211994ab546 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:35:30 +0000 Subject: [PATCH 51/61] Oban queue simplification --- changelog.d/oban-queues.change | 1 + config/config.exs | 12 +------ lib/pleroma/scheduled_activity.ex | 2 +- lib/pleroma/web/federator.ex | 2 +- .../workers/attachments_cleanup_worker.ex | 2 +- lib/pleroma/workers/backup_worker.ex | 2 +- .../workers/cron/new_users_digest_worker.ex | 2 +- lib/pleroma/workers/mailer_worker.ex | 2 +- lib/pleroma/workers/mute_expire_worker.ex | 2 +- lib/pleroma/workers/poll_worker.ex | 2 +- lib/pleroma/workers/purge_expired_activity.ex | 4 +-- lib/pleroma/workers/purge_expired_filter.ex | 4 +-- lib/pleroma/workers/purge_expired_token.ex | 2 +- lib/pleroma/workers/remote_fetcher_worker.ex | 2 +- .../workers/rich_media_expiration_worker.ex | 2 +- .../workers/scheduled_activity_worker.ex | 2 +- .../20240527144418_oban_queues_refactor.exs | 32 +++++++++++++++++++ 17 files changed, 50 insertions(+), 27 deletions(-) create mode 100644 changelog.d/oban-queues.change create mode 100644 priv/repo/migrations/20240527144418_oban_queues_refactor.exs diff --git a/changelog.d/oban-queues.change b/changelog.d/oban-queues.change new file mode 100644 index 0000000000..16df6409a0 --- /dev/null +++ b/changelog.d/oban-queues.change @@ -0,0 +1 @@ +Oban queues have refactored to simplify the queue design diff --git a/config/config.exs b/config/config.exs index b93de52e1c..b52021373b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -574,24 +574,14 @@ log: false, queues: [ activity_expiration: 10, - token_expiration: 5, - filter_expiration: 1, - backup: 1, federator_incoming: 5, federator_outgoing: 5, ingestion_queue: 50, web_push: 50, - mailer: 10, transmogrifier: 20, - scheduled_activities: 10, - poll_notifications: 10, background: 5, - remote_fetcher: 2, - attachments_cleanup: 1, - new_users_digest: 1, - mute_expire: 5, search_indexing: [limit: 10, paused: true], - rich_media_expiration: 2 + slow: 1 ], plugins: [Oban.Plugins.Pruner], crontab: [ diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 63c6cb45b1..c361d7d893 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -204,7 +204,7 @@ def due_activities(offset \\ 0) do def job_query(scheduled_activity_id) do from(j in Oban.Job, - where: j.queue == "scheduled_activities", + where: j.queue == "federator_outgoing", where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) ) end diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 1f2c3835a8..4b30fd21d2 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -44,7 +44,7 @@ def incoming_ap_doc(%{params: params, req_headers: req_headers}) do end def incoming_ap_doc(%{"type" => "Delete"} = params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) end def incoming_ap_doc(params) do diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 4c17640537..0b570b70b0 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo - use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" + use Pleroma.Workers.WorkerHelper, queue: "slow" @impl Oban.Worker def perform(%Job{ diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index a485ddb4b4..54ac31a3c6 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do - use Oban.Worker, queue: :backup, max_attempts: 1 + use Oban.Worker, queue: :slow, max_attempts: 1 alias Oban.Job alias Pleroma.User.Backup diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 1c3e445aa9..d2abb2d3b1 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(_job) do diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 940716558f..652bf77e01 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index 8ce458d488..8ad287a7f9 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MuteExpireWorker do - use Pleroma.Workers.WorkerHelper, queue: "mute_expire" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index 022d026f87..70df541931 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do @moduledoc """ Generates notifications when a poll ends. """ - use Pleroma.Workers.WorkerHelper, queue: "poll_notifications" + use Pleroma.Workers.WorkerHelper, queue: "background" alias Pleroma.Activity alias Pleroma.Notification diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index e554684feb..a65593b6e5 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do Worker which purges expired activity. """ - use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -59,7 +59,7 @@ defp find_user(ap_id) do def get_expiration(id) do from(j in Oban.Job, where: j.state == "scheduled", - where: j.queue == "activity_expiration", + where: j.queue == "slow", where: fragment("?->>'activity_id' = ?", j.args, ^id) ) |> Pleroma.Repo.one() diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 9114aeb7f4..1f6931e4c2 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do Worker which purges expired filters """ - use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -38,7 +38,7 @@ def timeout(_job), do: :timer.seconds(5) def get_expiration(id) do from(j in Job, where: j.state == "scheduled", - where: j.queue == "filter_expiration", + where: j.queue == "background", where: fragment("?->'filter_id' = ?", j.args, ^id) ) |> Repo.one() diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 2ccd9e80b9..1854bf5619 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do Worker which purges expired OAuth tokens """ - use Oban.Worker, queue: :token_expiration, max_attempts: 1 + use Oban.Worker, queue: :background, max_attempts: 1 @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index c264184833..ed04c54b2b 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do diff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_expiration_worker.ex index d7ae497a71..0b74687cfd 100644 --- a/lib/pleroma/workers/rich_media_expiration_worker.ex +++ b/lib/pleroma/workers/rich_media_expiration_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do alias Pleroma.Web.RichMedia.Card use Oban.Worker, - queue: :rich_media_expiration + queue: :background @impl Oban.Worker def perform(%Job{args: %{"url" => url} = _args}) do diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 4df84d00f9..ab62686f42 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do The worker to post scheduled activity. """ - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" alias Pleroma.Repo alias Pleroma.ScheduledActivity diff --git a/priv/repo/migrations/20240527144418_oban_queues_refactor.exs b/priv/repo/migrations/20240527144418_oban_queues_refactor.exs new file mode 100644 index 0000000000..64ee28dfd3 --- /dev/null +++ b/priv/repo/migrations/20240527144418_oban_queues_refactor.exs @@ -0,0 +1,32 @@ +defmodule Pleroma.Repo.Migrations.ObanQueuesRefactor do + use Ecto.Migration + + @changed_queues [ + {"attachments_cleanup", "slow"}, + {"mailer", "background"}, + {"mute_expire", "background"}, + {"poll_notifications", "background"}, + {"activity_expiration", "slow"}, + {"filter_expiration", "background"}, + {"token_expiration", "background"}, + {"remote_fetcher", "background"}, + {"rich_media_expiration", "background"} + ] + + def up do + Enum.each(@changed_queues, fn {old, new} -> + execute("UPDATE oban_jobs SET queue = '#{new}' WHERE queue = '#{old}';") + end) + + # Handled special as reverting this would not be ideal and leaving it is harmless + execute( + "UPDATE oban_jobs SET queue = 'federator_outgoing' WHERE queue = 'scheduled_activities';" + ) + end + + def down do + # Just move all slow queue jobs to background queue if we are reverting + # as the slow queue will not be processing jobs + execute("UPDATE oban_jobs SET queue = 'background' WHERE queue = 'slow';") + end +end From f63e44b8bc8e4e2f21fe21f1407a85d072dcab6d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 13:46:15 -0400 Subject: [PATCH 52/61] Fix Oban related tests --- test/pleroma/scheduled_activity_test.exs | 3 +-- .../scheduled_activity_controller_test.exs | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/test/pleroma/scheduled_activity_test.exs b/test/pleroma/scheduled_activity_test.exs index 4818e8bcf1..aaf643cfc5 100644 --- a/test/pleroma/scheduled_activity_test.exs +++ b/test/pleroma/scheduled_activity_test.exs @@ -31,8 +31,7 @@ test "scheduled activities with jobs when ScheduledActivity enabled" do {:ok, sa1} = ScheduledActivity.create(user, attrs) {:ok, sa2} = ScheduledActivity.create(user, attrs) - jobs = - Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + jobs = Repo.all(from(j in Oban.Job, where: j.queue == "federator_outgoing", select: j.args)) assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] end diff --git a/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index 632242221e..2d6b2aee23 100644 --- a/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.Web.ConnCase, async: true alias Pleroma.Repo @@ -78,7 +79,7 @@ test "updates a scheduled activity" do } ) - job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + job = Repo.one(from(j in Oban.Job, where: j.queue == "federator_outgoing")) assert job.args == %{"activity_id" => scheduled_activity.id} assert DateTime.truncate(job.scheduled_at, :second) == to_datetime(scheduled_at) @@ -124,9 +125,11 @@ test "deletes a scheduled activity" do } ) - job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) - - assert job.args == %{"activity_id" => scheduled_activity.id} + assert_enqueued( + worker: Pleroma.Workers.ScheduledActivityWorker, + args: %{"activity_id" => scheduled_activity.id}, + queue: :federator_outgoing + ) res_conn = conn @@ -135,7 +138,11 @@ test "deletes a scheduled activity" do assert %{} = json_response_and_validate_schema(res_conn, 200) refute Repo.get(ScheduledActivity, scheduled_activity.id) - refute Repo.get(Oban.Job, job.id) + + refute_enqueued( + worker: Pleroma.Workers.ScheduledActivityWorker, + args: %{"activity_id" => scheduled_activity.id} + ) res_conn = conn From 29eac86dc0bb246e983afe4209332194bf11bed0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 13:53:22 -0400 Subject: [PATCH 53/61] Logger metadata changelog --- changelog.d/logger-metadata.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/logger-metadata.add diff --git a/changelog.d/logger-metadata.add b/changelog.d/logger-metadata.add new file mode 100644 index 0000000000..6c627a972d --- /dev/null +++ b/changelog.d/logger-metadata.add @@ -0,0 +1 @@ +Logger metadata is now attached to some logs to help with troubleshooting and analysis From 6b8c15a4a1fdbc2a2a4ef194d9519e717470c632 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:11:42 -0400 Subject: [PATCH 54/61] Remove MediaProxyWarmingPolicy config for ConcurrentLimiter as we are not using it --- config/config.exs | 1 - .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/config/config.exs b/config/config.exs index a025defeb3..3c35f439a8 100644 --- a/config/config.exs +++ b/config/config.exs @@ -902,7 +902,6 @@ config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, - {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}, {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index e5eb6896a1..0c5b53def0 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -22,11 +22,7 @@ defp prefetch(url) do Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") - if Pleroma.Config.get(:env) == :test do - fetch(prefetch_url) - else - Task.start(fn -> fetch(prefetch_url) end) - end + fetch(prefetch_url) end end From ba511a30b923cc9248686702f75a4041d69c7bee Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:12:38 -0400 Subject: [PATCH 55/61] RichMedia use of ConcurrentLimiter was removed in the refactor --- config/config.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 3c35f439a8..e7e751d8af 100644 --- a/config/config.exs +++ b/config/config.exs @@ -901,7 +901,6 @@ process_chunk_size: 100 config :pleroma, ConcurrentLimiter, [ - {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] From 81e44ced0c7251b5a6b585f297e1e00fad08c6d1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 22:13:20 +0400 Subject: [PATCH 56/61] HTTPSecurityPlug: Fix tests --- config/test.exs | 2 +- lib/pleroma/web/plugs/http_security_plug.ex | 4 ++-- test/pleroma/web/plugs/http_security_plug_test.exs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/test.exs b/config/test.exs index b5c9c6e4a0..6c88ad3c68 100644 --- a/config/test.exs +++ b/config/test.exs @@ -154,7 +154,7 @@ config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock -config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a1dc6c02ae..38f6c511e3 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -116,7 +116,7 @@ defp csp_string do end connect_src = - if @config_impl.get(:env) == :dev do + if @config_impl.get([:env]) == :dev do [connect_src, " http://localhost:3035/"] else connect_src @@ -124,7 +124,7 @@ defp csp_string do script_src = if @config_impl.get([:http_security, :allow_unsafe_eval]) do - if @config_impl.get(:env) == :dev do + if @config_impl.get([:env]) == :dev do "script-src 'self' 'unsafe-eval'" else "script-src 'self' 'wasm-unsafe-eval'" diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index 80ad1fa7d2..11a351a41f 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -15,7 +15,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do end defp mock_config(config, additional \\ %{}) do - Pleroma.UnstubbedConfigMock + Pleroma.StaticStubbedConfigMock |> stub(:get, fn [:http_security, key] -> config[key] key -> additional[key] From bb86a01b9b5889155f60f08143755dd101d925f0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 15:20:47 -0400 Subject: [PATCH 57/61] Credo --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index d2b2cae0bf..e6161455dc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -522,7 +522,7 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end - defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do + defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do Logger.metadata(actor: actor, type: type) conn end From 73d58c22d4a9a87539cf1c3a33083464fc4b8540 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 08:09:19 +0400 Subject: [PATCH 58/61] Linting --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index d2b2cae0bf..e6161455dc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -522,7 +522,7 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end - defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do + defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do Logger.metadata(actor: actor, type: type) conn end From f5978da67633110acbc6494ff6f07b3f07424779 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 14:00:25 +0400 Subject: [PATCH 59/61] HTTPSignaturePlugTest: Rewrite to use mox. --- config/test.exs | 4 + lib/pleroma/http_signatures_api.ex | 4 + lib/pleroma/web/plugs/http_signature_plug.ex | 19 +- .../web/plugs/http_signature_plug_test.exs | 219 +++++++++--------- test/support/data_case.ex | 1 + test/support/http_signatures_proxy.ex | 9 + test/support/mocks.ex | 1 + 7 files changed, 144 insertions(+), 113 deletions(-) create mode 100644 lib/pleroma/http_signatures_api.ex create mode 100644 test/support/http_signatures_proxy.ex diff --git a/config/test.exs b/config/test.exs index 6c88ad3c68..0d4c82e0ef 100644 --- a/config/test.exs +++ b/config/test.exs @@ -155,6 +155,10 @@ config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock + +config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, + http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/http_signatures_api.ex b/lib/pleroma/http_signatures_api.ex new file mode 100644 index 0000000000..8e73dc98ef --- /dev/null +++ b/lib/pleroma/http_signatures_api.ex @@ -0,0 +1,4 @@ +defmodule Pleroma.HTTPSignaturesAPI do + @callback validate_conn(conn :: Plug.Conn.t()) :: boolean + @callback signature_for_conn(conn :: Plug.Conn.t()) :: map +end diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index 71b2a5f51f..6bf2dd432a 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -8,11 +8,17 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] - alias Pleroma.Config alias Pleroma.Web.ActivityPub.MRF require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @http_signatures_impl Application.compile_env( + :pleroma, + [__MODULE__, :http_signatures_impl], + HTTPSignatures + ) + def init(options) do options end @@ -41,7 +47,7 @@ defp validate_signature(conn, request_target) do |> put_req_header("(request-target)", request_target) |> put_req_header("@request-target", request_target) - HTTPSignatures.validate_conn(conn) + @http_signatures_impl.validate_conn(conn) end defp validate_signature(conn) do @@ -108,9 +114,9 @@ defp has_signature_header?(conn) do defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do - if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do exceptions = - Pleroma.Config.get([:activitypub, :authorized_fetch_mode_exceptions], []) + @config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], []) |> Enum.map(&InetHelper.parse_cidr/1) if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do @@ -129,7 +135,8 @@ defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do defp maybe_filter_requests(%{halted: true} = conn), do: conn defp maybe_filter_requests(conn) do - if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and + conn.assigns[:actor_id] do %{host: host} = URI.parse(conn.assigns.actor_id) if MRF.subdomain_match?(rejected_domains(), host) do @@ -145,7 +152,7 @@ defp maybe_filter_requests(conn) do end defp rejected_domains do - Config.get([:instance, :rejected_instances]) + @config_impl.get([:instance, :rejected_instances]) |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() end diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index b871d956e6..5f049dc45e 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -3,89 +3,88 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do - use Pleroma.Web.ConnCase + use Pleroma.Web.ConnCase, async: true alias Pleroma.Web.Plugs.HTTPSignaturePlug + alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock + alias Pleroma.StaticStubbedConfigMock, as: ConfigMock import Plug.Conn import Phoenix.Controller, only: [put_format: 2] - import Mock + import Mox - test "it call HTTPSignatures to check validity if the actor signed it" do + test "it calls HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> put_format("activity+json") - |> HTTPSignaturePlug.call(%{}) + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) - assert conn.assigns.valid_signature == true - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end + conn = + conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false end describe "requires a signature when `authorized_fetch_mode` is enabled" do setup do - clear_config([:activitypub, :authorized_fetch_mode], true) - params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) |> put_format("activity+json") [conn: conn] end - test "when signature header is present", %{conn: conn} do - with_mock HTTPSignatures, - validate_conn: fn _ -> false end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) + test "when signature header is present", %{conn: orig_conn} do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end) - assert conn.assigns.valid_signature == false - assert conn.halted == true - assert conn.status == 401 - assert conn.state == :sent - assert conn.resp_body == "Request not signed" - assert called(HTTPSignatures.validate_conn(:_)) - end + HTTPSignaturesMock + |> expect(:validate_conn, 2, fn _ -> false end) - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) + conn = + orig_conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) - assert conn.assigns.valid_signature == true - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end + assert conn.assigns.valid_signature == false + assert conn.halted == true + assert conn.status == 401 + assert conn.state == :sent + assert conn.resp_body == "Request not signed" + + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + orig_conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false end test "halts the connection when `signature` header is not present", %{conn: conn} do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end) + conn = HTTPSignaturePlug.call(conn, %{}) assert conn.assigns[:valid_signature] == nil assert conn.halted == true @@ -95,65 +94,71 @@ test "halts the connection when `signature` header is not present", %{conn: conn end test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do - clear_config([:activitypub, :authorized_fetch_mode_exceptions], ["192.168.0.0/24"]) + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> + ["192.168.0.0/24"] + end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) - with_mock HTTPSignatures, validate_conn: fn _ -> false end do - conn = - conn - |> Map.put(:remote_ip, {192, 168, 0, 1}) - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) + HTTPSignaturesMock + |> expect(:validate_conn, 2, fn _ -> false end) - assert conn.remote_ip == {192, 168, 0, 1} - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end - end - end - - test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do - clear_config([:activitypub, :authorized_fetch_mode], true) - clear_config([:instance, :rejected_instances], [{"mastodon.example.org", "no reason"}]) - - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do conn = - build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + conn + |> Map.put(:remote_ip, {192, 168, 0, 1}) |> put_req_header( "signature", "keyId=\"http://mastodon.example.org/users/admin#main-key" ) - |> put_format("activity+json") |> HTTPSignaturePlug.call(%{}) - assert conn.assigns.valid_signature == true - assert conn.halted == true - assert called(HTTPSignatures.validate_conn(:_)) - end - - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://allowed.example.org/users/admin#main-key"} - end do - conn = - build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) - |> put_req_header( - "signature", - "keyId=\"http://allowed.example.org/users/admin#main-key" - ) - |> put_format("activity+json") - |> HTTPSignaturePlug.call(%{}) - - assert conn.assigns.valid_signature == true + assert conn.remote_ip == {192, 168, 0, 1} assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) end end + + test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:instance, :rejected_instances] -> + [{"mastodon.example.org", "no reason"}] + end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == true + + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:instance, :rejected_instances] -> + [{"mastodon.example.org", "no reason"}] + end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://allowed.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false + end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 14403f0b81..52d4bef1a1 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -116,6 +116,7 @@ def stub_pipeline do Mox.stub_with(Pleroma.Web.FederatorMock, Pleroma.Web.Federator) Mox.stub_with(Pleroma.ConfigMock, Pleroma.Config) Mox.stub_with(Pleroma.StaticStubbedConfigMock, Pleroma.Test.StaticConfig) + Mox.stub_with(Pleroma.StubbedHTTPSignaturesMock, Pleroma.Test.HTTPSignaturesProxy) end def ensure_local_uploader(context) do diff --git a/test/support/http_signatures_proxy.ex b/test/support/http_signatures_proxy.ex new file mode 100644 index 0000000000..4c6b39d19c --- /dev/null +++ b/test/support/http_signatures_proxy.ex @@ -0,0 +1,9 @@ +defmodule Pleroma.Test.HTTPSignaturesProxy do + @behaviour Pleroma.HTTPSignaturesAPI + + @impl true + defdelegate validate_conn(conn), to: HTTPSignatures + + @impl true + defdelegate signature_for_conn(conn), to: HTTPSignatures +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index d906f0e1da..63cbc49ab6 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -28,6 +28,7 @@ Mox.defmock(Pleroma.ConfigMock, for: Pleroma.Config.Getting) Mox.defmock(Pleroma.UnstubbedConfigMock, for: Pleroma.Config.Getting) Mox.defmock(Pleroma.StaticStubbedConfigMock, for: Pleroma.Config.Getting) +Mox.defmock(Pleroma.StubbedHTTPSignaturesMock, for: Pleroma.HTTPSignaturesAPI) Mox.defmock(Pleroma.LoggerMock, for: Pleroma.Logging) From 8066645f711f38986b3d0f9c0b34d6956563da6a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 14:20:48 +0400 Subject: [PATCH 60/61] Linting --- test/pleroma/web/plugs/http_signature_plug_test.exs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 5f049dc45e..9d07270bb4 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -4,13 +4,14 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do use Pleroma.Web.ConnCase, async: true - alias Pleroma.Web.Plugs.HTTPSignaturePlug - alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock - alias Pleroma.StaticStubbedConfigMock, as: ConfigMock - import Plug.Conn - import Phoenix.Controller, only: [put_format: 2] + alias Pleroma.StaticStubbedConfigMock, as: ConfigMock + alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock + alias Pleroma.Web.Plugs.HTTPSignaturePlug + import Mox + import Phoenix.Controller, only: [put_format: 2] + import Plug.Conn test "it calls HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} From 335691bae1a002c1a3bec956884fe665114285ec Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 14:38:44 +0400 Subject: [PATCH 61/61] Add changelog --- changelog.d/authorized-fetch-rejections.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/authorized-fetch-rejections.add diff --git a/changelog.d/authorized-fetch-rejections.add b/changelog.d/authorized-fetch-rejections.add new file mode 100644 index 0000000000..66e15a979b --- /dev/null +++ b/changelog.d/authorized-fetch-rejections.add @@ -0,0 +1 @@ +Add an option to reject certain domains when authorized fetch is enabled.