bigbuffet/pleroma
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/develop' into fork

Browse source 
Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
This commit is contained in:
marcin mikołajczak 2024-05-28 16:22:38 +02:00
commit a0eff7cc94
72 changed files with 1374 additions and 314 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/3904.security Normal file
View file

@ -0,0 +1 @@
HTTP Security: By default, don't allow unsafe-eval. The setting needs to be changed to allow Flash emulation.

0
changelog.d/3907.skip Normal file
View file

1
changelog.d/auth-fetch-exception.add Normal file
View file

@ -0,0 +1 @@
HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions configuration

1
changelog.d/authorized-fetch-rejections.add Normal file
View file

@ -0,0 +1 @@
Add an option to reject certain domains when authorized fetch is enabled.

1
changelog.d/logger-metadata.add Normal file
View file

@ -0,0 +1 @@
Logger metadata is now attached to some logs to help with troubleshooting and analysis

1
changelog.d/mediaproxy-http.fix Normal file
View file

@ -0,0 +1 @@
Ensure MediaProxy HTTP requests obey all the defined connection settings

1
changelog.d/oban-queues.change Normal file
View file

@ -0,0 +1 @@
Oban queues have refactored to simplify the queue design

1
changelog.d/pools.change Normal file
View file

@ -0,0 +1 @@
HTTP connection pool adjustments

1
changelog.d/qdrant_search.add Normal file
View file

@ -0,0 +1 @@
Add Qdrant/OpenAI embedding search

1
changelog.d/support-honk-image-summaries.add Normal file
View file

@ -0,0 +1 @@
Support honk-style attachment summaries as alt-text.

59
config/config.exs
View file

@ -135,13 +135,13 @@
config :logger, :console,
level: :debug,
format: "\n$time $metadata[$level] $message\n",
metadata: [:request_id]
metadata: [:actor, :path, :type, :user]
config :logger, :ex_syslogger,
level: :debug,
ident: "pleroma",
format: "$metadata[$level] $message",
metadata: [:request_id]
metadata: [:actor, :path, :type, :user]
config :mime, :types, %{
"application/xml" => ["xml"],
@ -192,6 +192,7 @@
allow_relay: true,
public: true,
quarantined_instances: [],
rejected_instances: [],
static_dir: "instance/static/",
allowed_post_formats: [
"text/plain",
@ -529,7 +530,8 @@
sts: false,
sts_max_age: 31_536_000,
ct_max_age: 2_592_000,
referrer_policy: "same-origin"
referrer_policy: "same-origin",
allow_unsafe_eval: false
config :cors_plug,
max_age: 86_400,
@ -591,26 +593,15 @@
log: false,
queues: [
activity_expiration: 10,
token_expiration: 5,
filter_expiration: 1,
backup: 1,
federator_incoming: 50,
federator_outgoing: 50,
federator_incoming: 5,
federator_outgoing: 5,
ingestion_queue: 50,
web_push: 50,
mailer: 10,
transmogrifier: 20,
scheduled_activities: 10,
poll_notifications: 10,
notifications: 20,
background: 5,
remote_fetcher: 2,
attachments_cleanup: 1,
new_users_digest: 1,
mute_expire: 5,
search_indexing: [limit: 10, paused: true],
rich_media_expiration: 2,
check_domain_resolve: 1
slow: 1
],
plugins: [Oban.Plugins.Pruner],
crontab: [
@ -860,22 +851,27 @@
config :pleroma, :pools,
federation: [
size: 50,
max_waiting: 10,
size: 75,
max_waiting: 20,
recv_timeout: 10_000
],
media: [
size: 50,
size: 75,
max_waiting: 20,
recv_timeout: 15_000
],
rich_media: [
size: 25,
max_waiting: 20,
recv_timeout: 15_000
],
upload: [
size: 25,
max_waiting: 5,
max_waiting: 20,
recv_timeout: 15_000
],
default: [
size: 10,
size: 50,
max_waiting: 2,
recv_timeout: 5_000
]
@ -889,6 +885,10 @@
max_connections: 50,
timeout: 150_000
],
rich_media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
@ -934,8 +934,6 @@
process_chunk_size: 100
config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
{Pleroma.Search, [max_running: 30, max_waiting: 50]},
{Pleroma.Webhook.Notify, [max_running: 5, max_waiting: 200]}
]
@ -978,6 +976,19 @@
config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000
config :pleroma, Pleroma.Search.QdrantSearch,
qdrant_url: "http://127.0.0.1:6333/",
qdrant_api_key: "",
openai_url: "http://127.0.0.1:11345",
# The healthcheck url has to be set to nil when used with the real openai
# API, as it doesn't have a healthcheck endpoint.
openai_healthcheck_url: "http://127.0.0.1:11345/health",
openai_model: "snowflake/snowflake-arctic-embed-xs",
openai_api_key: "",
qdrant_index_configuration: %{
vectors: %{size: 384, distance: "Cosine"}
}
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

18
config/description.exs
View file

@ -774,6 +774,18 @@
{"*.quarantined.com", "Reason"}
]
},
%{
key: :rejected_instances,
type: {:list, :tuple},
key_placeholder: "instance",
value_placeholder: "reason",
description:
"List of ActivityPub instances to reject requests from if authorized_fetch_mode is enabled",
suggestions: [
{"rejected.com", "Reason"},
{"*.rejected.com", "Reason"}
]
},
%{
key: :static_dir,
type: :string,
@ -1848,6 +1860,12 @@
type: :boolean,
description: "Require HTTP signatures for AP fetches"
},
%{
key: :authorized_fetch_mode_exceptions,
type: {:list, :string},
description:
"List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)"
},
%{
key: :note_replies_output_limit,
type: :integer,

4
config/dev.exs
View file

@ -35,8 +35,8 @@
# configured to run both http and https servers on
# different ports.
# Do not include metadata nor timestamps in development logs
config :logger, :console, format: "[$level] $message\n"
# Do not include timestamps in development logs
config :logger, :console, format: "$metadata[$level] $message\n"
# Set a higher stacktrace during development. Avoid configuring such
# in production as building large stacktraces may be expensive.

5
config/test.exs
View file

@ -166,6 +166,11 @@
config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock
config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock
config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug,
http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock
peer_module =
if String.to_integer(System.otp_release()) >= 25 do

3
docs/configuration/cheatsheet.md
View file

@ -36,6 +36,7 @@ To add configuration to your config file, you can copy it from the base config.
* `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance.
* `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
* `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send.
* `rejected_instances`: ActivityPub instances to reject requests from if authorized_fetch_mode is enabled.
* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
older software for theses nicknames.
@ -283,6 +284,7 @@ Notes:
* `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question
* `sign_object_fetches`: Sign object fetches with HTTP signatures
* `authorized_fetch_mode`: Require HTTP signatures for AP fetches
* `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)
## Pleroma.User
@ -471,6 +473,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent.
* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
* `allow_unsafe_eval`: Adds `wasm-unsafe-eval` to the CSP header. Needed for some non-essential frontend features like Flash emulation.
### Pleroma.Web.Plugs.RemoteIp

24
docs/configuration/search.md
View file

@ -10,6 +10,30 @@ To use built-in search that has no external dependencies, set the search module
While it has no external dependencies, it has problems with performance and relevancy.
## QdrantSearch
This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.
The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`:
> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch
Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions.
You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities.
### Indexing and model options
To see the available configuration options, check out the QdrantSearch section in `config/config.exs`.
The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail.
Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`.
E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024.
When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other.
## Meilisearch
Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million

80
lib/mix/tasks/pleroma/search/indexer.ex Normal file
View file

@ -0,0 +1,80 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Search.Indexer do
import Mix.Pleroma
import Ecto.Query
alias Pleroma.Workers.SearchIndexingWorker
def run(["create_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
IO.puts("Index created")
else
e -> IO.puts("Could not create index: #{inspect(e)}")
end
end
def run(["drop_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
IO.puts("Index dropped")
else
e -> IO.puts("Could not drop index: #{inspect(e)}")
end
end
def run(["index" | options]) do
{options, [], []} =
OptionParser.parse(
options,
strict: [
limit: :integer
]
)
start_pleroma()
limit = Keyword.get(options, :limit, 100_000)
per_step = 1000
chunks = max(div(limit, per_step), 1)
1..chunks
|> Enum.each(fn step ->
q =
from(a in Pleroma.Activity,
limit: ^per_step,
offset: ^per_step * (^step - 1),
select: [:id],
order_by: [desc: :id]
)
{:ok, ids} =
Pleroma.Repo.transaction(fn ->
Pleroma.Repo.stream(q, timeout: :infinity)
|> Enum.map(fn a ->
a.id
end)
end)
IO.puts("Got #{length(ids)} activities, adding to indexer")
ids
|> Enum.chunk_every(100)
|> Enum.each(fn chunk ->
IO.puts("Adding #{length(chunk)} activities to indexing queue")
chunk
|> Enum.map(fn id ->
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
end)
|> Oban.insert_all()
end)
end)
end
end

7
lib/pleroma/application.ex
View file

@ -15,6 +15,7 @@ defmodule Pleroma.Application do
@compat_name Mix.Project.config()[:compat_name]
@version Mix.Project.config()[:version]
@repository Mix.Project.config()[:source_url]
@compile_env Mix.env()
def name, do: @name
def compat_name, do: @compat_name
@ -54,7 +55,11 @@ def start(_type, _args) do
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn()
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
if @compile_env != :test do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
Pleroma.ApplicationRequirements.verify!()
load_custom_modules()
Pleroma.Docs.JSON.compile()

11
lib/pleroma/helpers/inet_helper.ex
View file

@ -16,4 +16,15 @@ def parse_address(ip) when is_binary(ip) do
def parse_address(ip) do
:inet.parse_address(ip)
end
def parse_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse_cidr!(proxy, true)
end
end

4
lib/pleroma/http_signatures_api.ex Normal file
View file

@ -0,0 +1,4 @@
defmodule Pleroma.HTTPSignaturesAPI do
@callback validate_conn(conn :: Plug.Conn.t()) :: boolean
@callback signature_for_conn(conn :: Plug.Conn.t()) :: map
end

2
lib/pleroma/scheduled_activity.ex
View file

@ -204,7 +204,7 @@ def due_activities(offset \\ 0) do
def job_query(scheduled_activity_id) do
from(j in Oban.Job,
where: j.queue == "scheduled_activities",
where: j.queue == "federator_outgoing",
where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
)
end

6
lib/pleroma/search/database_search.ex
View file

@ -48,6 +48,12 @@ def add_to_index(_activity), do: :ok
@impl true
def remove_from_index(_object), do: :ok
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
@impl true
def healthcheck_endpoints, do: nil

6
lib/pleroma/search/meilisearch.ex
View file

@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do
@behaviour Pleroma.Search.SearchBackend
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
defp meili_headers do
private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])

182
lib/pleroma/search/qdrant_search.ex Normal file
View file

@ -0,0 +1,182 @@
defmodule Pleroma.Search.QdrantSearch do
@behaviour Pleroma.Search.SearchBackend
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias __MODULE__.OpenAIClient
alias __MODULE__.QdrantClient
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
@impl true
def create_index do
payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
:ok
else
e -> {:error, e}
end
end
@impl true
def drop_index do
with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
:ok
else
e -> {:error, e}
end
end
def get_embedding(text) do
with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
OpenAIClient.post("/v1/embeddings", %{
input: text,
model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
}) do
{:ok, embedding}
else
_ ->
{:error, "Failed to get embedding"}
end
end
defp actor_from_activity(%{data: %{"actor" => actor}}) do
actor
end
defp actor_from_activity(_), do: nil
defp build_index_payload(activity, embedding) do
actor = actor_from_activity(activity)
published_at = activity.data["published"]
%{
points: [
%{
id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
vector: embedding,
payload: %{actor: actor, published_at: published_at}
}
]
}
end
defp build_search_payload(embedding, options) do
base = %{
vector: embedding,
limit: options[:limit] || 20,
offset: options[:offset] || 0
}
if author = options[:author] do
Map.put(base, :filter, %{
must: [%{key: "actor", match: %{value: author.ap_id}}]
})
else
base
end
end
@impl true
def add_to_index(activity) do
# This will only index public or unlisted notes
maybe_search_data = object_to_search_data(activity.object)
if activity.data["type"] == "Create" and maybe_search_data do
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
else
:ok
end
end
@impl true
def remove_from_index(object) do
activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
with {:ok, %{status: 200}} <-
QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
:ok
else
e -> {:error, e}
end
end
@impl true
def search(user, original_query, options) do
query = "Represent this sentence for searching relevant passages: #{original_query}"
with {:ok, embedding} <- get_embedding(query),
{:ok, %{body: %{"result" => result}}} <-
QdrantClient.post(
"/collections/posts/points/search",
build_search_payload(embedding, options)
) do
ids =
Enum.map(result, fn %{"id" => id} ->
Ecto.UUID.dump!(id)
end)
from(a in Activity, where: a.id in ^ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
|> Pleroma.Repo.all()
|> maybe_fetch(user, original_query)
else
_ ->
[]
end
end
@impl true
def healthcheck_endpoints do
qdrant_health =
Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
|> URI.parse()
|> Map.put(:path, "/healthz")
|> URI.to_string()
openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
[qdrant_health, openai_health] |> Enum.filter(& &1)
end
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"Authorization",
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
])
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
end

10
lib/pleroma/search/search_backend.ex
View file

@ -22,6 +22,16 @@ defmodule Pleroma.Search.SearchBackend do
"""
@callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
@doc """
Create the index
"""
@callback create_index() :: :ok | {:error, any()}
@doc """
Drop the index
"""
@callback drop_index() :: :ok | {:error, any()}
@doc """
Healthcheck endpoints of search backend infrastructure to monitor for controlling
processing of jobs in the Oban queue.

8
lib/pleroma/web/activity_pub/activity_pub_controller.ex
View file

@ -62,6 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
when action in [:following, :followers, :pinned, :inbox, :outbox, :update_outbox]
)
plug(:log_inbox_metadata when action in [:inbox])
plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay])
@ -531,6 +532,13 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do
conn
end
defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
Logger.metadata(actor: actor, type: type)
conn
end
defp log_inbox_metadata(conn, _), do: conn
def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
with {:ok, object} <-
ActivityPub.upload(

18
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
View file

@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@adapter_options [
pool: :media,
recv_timeout: 10_000
]
@impl true
def history_awareness, do: :auto
@ -27,17 +22,14 @@ defp prefetch(url) do
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
if Pleroma.Config.get(:env) == :test do
fetch(prefetch_url)
else
ConcurrentLimiter.limit(__MODULE__, fn ->
Task.start(fn -> fetch(prefetch_url) end)
end)
end
fetch(prefetch_url)
end
end
defp fetch(url), do: HTTP.get(url, [], @adapter_options)
defp fetch(url) do
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
HTTP.get(url, [], http_client_opts)
end
defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
Enum.each(attachments, fn

3
lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
View file

@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
field(:type, :string, default: "Link")
field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
field(:name, :string)
field(:summary, :string)
field(:blurhash, :string)
embeds_many :url, UrlObjectValidator, primary_key: false do
@ -44,7 +45,7 @@ def changeset(struct, data) do
|> fix_url()
struct
|> cast(data, [:id, :type, :mediaType, :name, :blurhash])
|> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash])
|> cast_embed(:url, with: &url_changeset/2, required: true)
end

6
lib/pleroma/web/api_spec/schemas/attachment.ex
View file

@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do
pleroma: %Schema{
type: :object,
properties: %{
mime_type: %Schema{type: :string, description: "mime type of the attachment"}
mime_type: %Schema{type: :string, description: "mime type of the attachment"},
name: %Schema{
type: :string,
description: "Name of the attachment, typically the filename"
}
}
}
},

2
lib/pleroma/web/endpoint.ex
View file

@ -23,6 +23,8 @@ defmodule Pleroma.Web.Endpoint do
plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
plug(Pleroma.Web.Plugs.LoggerMetadataPath)
plug(Pleroma.Web.Plugs.SetLocalePlug)
plug(CORSPlug)
plug(Pleroma.Web.Plugs.HTTPSecurityPlug)

2
lib/pleroma/web/federator.ex
View file

@ -44,7 +44,7 @@ def incoming_ap_doc(%{params: params, req_headers: req_headers}) do
end
def incoming_ap_doc(%{"type" => "Delete"} = params) do
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3)
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow)
end
def incoming_ap_doc(params) do

7
lib/pleroma/web/mastodon_api/views/instance_view.ex
View file

@ -194,6 +194,7 @@ defp common_information(instance) do
def federation do
quarantined = Config.get([:instance, :quarantined_instances], [])
rejected = Config.get([:instance, :rejected_instances], [])
if Config.get([:mrf, :transparency]) do
{:ok, data} = MRF.describe()
@ -213,6 +214,12 @@ def federation do
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|> Map.new()
})
|> Map.put(
:rejected_instances,
rejected
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|> Map.new()
)
else
%{}
end

17
lib/pleroma/web/mastodon_api/views/status_view.ex
View file

@ -628,6 +628,19 @@ def render("attachment.json", %{attachment: attachment}) do
to_string(attachment["id"] || hash_id)
end
description =
if attachment["summary"] do
HTML.strip_tags(attachment["summary"])
else
attachment["name"]
end
name = if attachment["summary"], do: attachment["name"]
pleroma =
%{mime_type: media_type}
|> Maps.put_if_present(:name, name)
%{
id: attachment_id,
url: href,
@ -635,8 +648,8 @@ def render("attachment.json", %{attachment: attachment}) do
preview_url: href_preview,
text_url: href,
type: type,
description: attachment["name"],
pleroma: %{mime_type: media_type},
description: description,
pleroma: pleroma,
blurhash: attachment["blurhash"]
}
|> Maps.put_if_present(:meta, meta)

3
lib/pleroma/web/media_proxy/media_proxy_controller.ex
View file

@ -54,9 +54,10 @@ def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
defp handle_preview(conn, url) do
media_proxy_url = MediaProxy.url(url)
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
with {:ok, %{status: status} = head_response} when status in 200..299 <-
Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do
Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do
content_type = Tesla.get_header(head_response, "content-type")
content_length = Tesla.get_header(head_response, "content-length")
content_length = content_length && String.to_integer(content_length)

53
lib/pleroma/web/plugs/http_security_plug.ex
View file

@ -3,26 +3,27 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
alias Pleroma.Config
import Plug.Conn
require Logger
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
def init(opts), do: opts
def call(conn, _options) do
if Config.get([:http_security, :enabled]) do
if @config_impl.get([:http_security, :enabled]) do
conn
|> merge_resp_headers(headers())
|> maybe_send_sts_header(Config.get([:http_security, :sts]))
|> maybe_send_sts_header(@config_impl.get([:http_security, :sts]))
else
conn
end
end
def primary_frontend do
with %{"name" => frontend} <- Config.get([:frontends, :primary]),
available <- Config.get([:frontends, :available]),
with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]),
available <- @config_impl.get([:frontends, :available]),
%{} = primary_frontend <- Map.get(available, frontend) do
{:ok, primary_frontend}
end
@ -37,8 +38,8 @@ def custom_http_frontend_headers do
end
def headers do
referrer_policy = Config.get([:http_security, :referrer_policy])
report_uri = Config.get([:http_security, :report_uri])
referrer_policy = @config_impl.get([:http_security, :referrer_policy])
report_uri = @config_impl.get([:http_security, :report_uri])
custom_http_frontend_headers = custom_http_frontend_headers()
headers = [
@ -87,11 +88,11 @@ def headers do
@csp_start [Enum.join(static_csp_rules, ";") <> ";"]
defp csp_string do
scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme]
static_url = Pleroma.Web.Endpoint.static_url()
websocket_url = Pleroma.Web.Endpoint.websocket_url()
report_uri = Config.get([:http_security, :report_uri])
sentry_dsn = Config.get([:frontend_configurations, :soapbox_fe, "sentryDsn"])
report_uri = @config_impl.get([:http_security, :report_uri])
sentry_dsn = @config_impl.get([:frontend_configurations, :soapbox_fe, "sentryDsn"])
img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'"
@ -99,8 +100,8 @@ defp csp_string do
# Strict multimedia CSP enforcement only when MediaProxy is enabled
{img_src, media_src, connect_src} =
if Config.get([:media_proxy, :enabled]) &&
!Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
if @config_impl.get([:media_proxy, :enabled]) &&
!@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
sources = build_csp_multimedia_source_list()
{
@ -117,7 +118,7 @@ defp csp_string do
end
connect_src =
if Config.get(:env) == :dev do
if @config_impl.get([:env]) == :dev do
[connect_src, " http://localhost:3035/"]
else
connect_src
@ -131,10 +132,14 @@ defp csp_string do
end
script_src =
if Config.get(:env) == :dev do
"script-src 'self' 'unsafe-eval'"
if @config_impl.get([:http_security, :allow_unsafe_eval]) do
if @config_impl.get([:env]) == :dev do
"script-src 'self' 'unsafe-eval'"
else
"script-src 'self' 'wasm-unsafe-eval'"
end
else
"script-src 'self' 'wasm-unsafe-eval'"
"script-src 'self'"
end
report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
@ -170,11 +175,11 @@ defp build_csp_param_from_whitelist(url), do: url
defp build_csp_multimedia_source_list do
media_proxy_whitelist =
[:media_proxy, :whitelist]
|> Config.get()
|> @config_impl.get()
|> build_csp_from_whitelist([])
captcha_method = Config.get([Pleroma.Captcha, :method])
captcha_endpoint = Config.get([captcha_method, :endpoint])
captcha_method = @config_impl.get([Pleroma.Captcha, :method])
captcha_endpoint = @config_impl.get([captcha_method, :endpoint])
map_tile_server_endpoint = map_tile_server()
@ -184,7 +189,7 @@ defp build_csp_multimedia_source_list do
[Pleroma.Upload, :base_url],
[Pleroma.Uploaders.S3, :public_endpoint]
]
|> Enum.map(&Config.get/1)
|> Enum.map(&@config_impl.get/1)
[captcha_endpoint | base_endpoints]
|> Enum.map(&build_csp_param/1)
@ -195,7 +200,7 @@ defp build_csp_multimedia_source_list do
defp map_tile_server do
with tile_server when is_binary(tile_server) <-
Config.get([:frontend_configurations, :soapbox_fe, "tileServer"]),
@config_impl.get([:frontend_configurations, :soapbox_fe, "tileServer"]),
%{host: host} <- URI.parse(tile_server) do
["*.#{host}"]
else
@ -222,7 +227,7 @@ defp build_csp_param(url) when is_binary(url) do
end
def warn_if_disabled do
unless Config.get([:http_security, :enabled]) do
unless @config_impl.get([:http_security, :enabled]) do
Logger.warning("
.i;;;;i.
iYcviii;vXY:
@ -267,8 +272,8 @@ def warn_if_disabled do
end
defp maybe_send_sts_header(conn, true) do
max_age_sts = Config.get([:http_security, :sts_max_age])
max_age_ct = Config.get([:http_security, :ct_max_age])
max_age_sts = @config_impl.get([:http_security, :sts_max_age])
max_age_ct = @config_impl.get([:http_security, :ct_max_age])
merge_resp_headers(conn, [
{"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},

56
lib/pleroma/web/plugs/http_signature_plug.ex
View file

@ -3,14 +3,22 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
alias Pleroma.Helpers.InetHelper
import Plug.Conn
import Phoenix.Controller, only: [get_format: 1, text: 2]
alias Pleroma.Config
alias Pleroma.Web.ActivityPub.MRF
require Logger
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
@http_signatures_impl Application.compile_env(
:pleroma,
[__MODULE__, :http_signatures_impl],
HTTPSignatures
)
def init(options) do
options
end
@ -39,7 +47,7 @@ defp validate_signature(conn, request_target) do
|> put_req_header("(request-target)", request_target)
|> put_req_header("@request-target", request_target)
HTTPSignatures.validate_conn(conn)
@http_signatures_impl.validate_conn(conn)
end
defp validate_signature(conn) do
@ -105,28 +113,30 @@ defp has_signature_header?(conn) do
defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
defp maybe_require_signature(conn) do
cond do
get_ip(conn) in Config.get([:instance, :trusted_unsigned], []) ->
conn
|> assign(:valid_signature, true)
|> assign(:actor_id, Pleroma.Web.ActivityPub.Relay.ap_id())
defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do
exceptions =
@config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], [])
|> Enum.map(&InetHelper.parse_cidr/1)
Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) ->
if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do
conn
else
conn
|> put_status(:unauthorized)
|> text("Request not signed")
|> halt()
true ->
conn
end
else
conn
end
end
defp maybe_filter_requests(%{halted: true} = conn), do: conn
defp maybe_filter_requests(conn) do
if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and
conn.assigns[:actor_id] do
%{host: host} = URI.parse(conn.assigns.actor_id)
if MRF.subdomain_match?(rejected_domains(), host) do
@ -142,26 +152,8 @@ defp maybe_filter_requests(conn) do
end
defp rejected_domains do
Config.get([:instance, :rejected_instances], [])
@config_impl.get([:instance, :rejected_instances])
|> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
|> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
end
defp get_ip(conn) do
forwarded_for =
conn
|> Plug.Conn.get_req_header("x-forwarded-for")
|> List.first()
if forwarded_for do
forwarded_for
|> String.split(",")
|> Enum.map(&String.trim/1)
|> List.first()
else
conn.remote_ip
|> :inet_parse.ntoa()
|> to_string()
end
end
end

12
lib/pleroma/web/plugs/logger_metadata_path.ex Normal file
View file

@ -0,0 +1,12 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.LoggerMetadataPath do
def init(opts), do: opts
def call(conn, _) do
Logger.metadata(path: conn.request_path)
conn
end
end

18
lib/pleroma/web/plugs/logger_metadata_user.ex Normal file
View file

@ -0,0 +1,18 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.LoggerMetadataUser do
alias Pleroma.User
def init(opts), do: opts
def call(%{assigns: %{user: user = %User{}}} = conn, _) do
Logger.metadata(user: user.nickname)
conn
end
def call(conn, _) do
conn
end
end

14
lib/pleroma/web/plugs/remote_ip.ex
View file

@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
"""
alias Pleroma.Config
alias Pleroma.Helpers.InetHelper
import Plug.Conn
@behaviour Plug
@ -30,19 +31,8 @@ defp remote_ip_opts do
proxies =
Config.get([__MODULE__, :proxies], [])
|> Enum.concat(reserved)
|> Enum.map(&maybe_add_cidr/1)
|> Enum.map(&InetHelper.parse_cidr/1)
{headers, proxies}
end
defp maybe_add_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse_cidr!(proxy, true)
end
end

2
lib/pleroma/web/rich_media/helpers.ex
View file

@ -71,7 +71,7 @@ defp check_content_length(headers) do
defp http_options do
[
pool: :media,
pool: :rich_media,
max_body: Config.get([:rich_media, :max_body], 5_000_000)
]
end

8
lib/pleroma/web/router.ex
View file

@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do
pipeline :browser do
plug(:accepts, ["html"])
plug(:fetch_session)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :oauth do
@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do
plug(:fetch_session)
plug(:authenticate)
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :no_auth_or_privacy_expectations_api do
plug(:base_api)
plug(:after_auth)
plug(Pleroma.Web.Plugs.IdempotencyPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
# Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported)
@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do
pipeline :api do
plug(:expect_public_instance_or_user_authentication)
plug(:no_auth_or_privacy_expectations_api)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :authenticated_api do
plug(:expect_user_authentication)
plug(:no_auth_or_privacy_expectations_api)
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :admin_api do
@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
plug(Pleroma.Web.Plugs.UserIsStaffPlug)
plug(Pleroma.Web.Plugs.IdempotencyPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :require_admin do
@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do
plug(:browser)
plug(:authenticate)
plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :well_known do
@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do
pipeline :pleroma_api do
plug(:accepts, ["html", "json"])
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end
pipeline :mailbox_preview do

2
lib/pleroma/workers/attachments_cleanup_worker.ex
View file

@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
alias Pleroma.Object
alias Pleroma.Repo
use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
use Pleroma.Workers.WorkerHelper, queue: "slow"
@impl Oban.Worker
def perform(%Job{

2
lib/pleroma/workers/backup_worker.ex
View file

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :backup, max_attempts: 1
use Oban.Worker, queue: :slow, max_attempts: 1
alias Oban.Job
alias Pleroma.User.Backup

2
lib/pleroma/workers/cron/new_users_digest_worker.ex
View file

@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
import Ecto.Query
use Pleroma.Workers.WorkerHelper, queue: "mailer"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(_job) do

2
lib/pleroma/workers/mailer_worker.ex
View file

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.MailerWorker do
use Pleroma.Workers.WorkerHelper, queue: "mailer"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do

2
lib/pleroma/workers/mute_expire_worker.ex
View file

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.MuteExpireWorker do
use Pleroma.Workers.WorkerHelper, queue: "mute_expire"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do

2
lib/pleroma/workers/poll_worker.ex
View file

@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do
@moduledoc """
Generates notifications when a poll ends.
"""
use Pleroma.Workers.WorkerHelper, queue: "poll_notifications"
use Pleroma.Workers.WorkerHelper, queue: "background"
alias Pleroma.Activity
alias Pleroma.Notification

4
lib/pleroma/workers/purge_expired_activity.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
Worker which purges expired activity.
"""
use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity]
use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity]
import Ecto.Query
@ -59,7 +59,7 @@ defp find_user(ap_id) do
def get_expiration(id) do
from(j in Oban.Job,
where: j.state == "scheduled",
where: j.queue == "activity_expiration",
where: j.queue == "slow",
where: fragment("?->>'activity_id' = ?", j.args, ^id)
)
|> Pleroma.Repo.one()

4
lib/pleroma/workers/purge_expired_filter.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
Worker which purges expired filters
"""
use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity]
use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity]
import Ecto.Query
@ -38,7 +38,7 @@ def timeout(_job), do: :timer.seconds(5)
def get_expiration(id) do
from(j in Job,
where: j.state == "scheduled",
where: j.queue == "filter_expiration",
where: j.queue == "background",
where: fragment("?->'filter_id' = ?", j.args, ^id)
)
|> Repo.one()

2
lib/pleroma/workers/purge_expired_token.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do
Worker which purges expired OAuth tokens
"""
use Oban.Worker, queue: :token_expiration, max_attempts: 1
use Oban.Worker, queue: :background, max_attempts: 1
@spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) ::
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}

2
lib/pleroma/workers/remote_fetcher_worker.ex
View file

@ -5,7 +5,7 @@
defmodule Pleroma.Workers.RemoteFetcherWorker do
alias Pleroma.Object.Fetcher
use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher"
use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker
def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do

2
lib/pleroma/workers/rich_media_expiration_worker.ex
View file

@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do
alias Pleroma.Web.RichMedia.Card
use Oban.Worker,
queue: :rich_media_expiration
queue: :background
@impl Oban.Worker
def perform(%Job{args: %{"url" => url} = _args}) do

2
lib/pleroma/workers/scheduled_activity_worker.ex
View file

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do
The worker to post scheduled activity.
"""
use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities"
use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing"
alias Pleroma.Repo
alias Pleroma.ScheduledActivity

84
priv/gettext/config_descriptions.pot
View file

@ -5973,3 +5973,87 @@ msgstr ""
msgctxt "config label at :pleroma-:instance > :languages"
msgid "Languages"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config description at :pleroma-:mrf_emoji"
msgid "Reject or force-unlisted emojis whose URLs or names match a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html)."
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode"
msgid " A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_url"
msgid " A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config description at :pleroma-:mrf_emoji > :remove_shortcode"
msgid " A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config description at :pleroma-:mrf_emoji > :remove_url"
msgid " A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_chunk_size"
msgid "The number of activities to fetch in the backup job for each chunk."
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_wait_time"
msgid "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed."
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config label at :pleroma-:mrf_emoji"
msgid "MRF Emoji"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode"
msgid "Federated timeline removal shortcode"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_url"
msgid "Federated timeline removal url"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config label at :pleroma-:mrf_emoji > :remove_shortcode"
msgid "Remove shortcode"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config label at :pleroma-:mrf_emoji > :remove_url"
msgid "Remove url"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_chunk_size"
msgid "Process Chunk Size"
msgstr ""
#: lib/pleroma/docs/translator.ex:5
#, elixir-autogen, elixir-format
msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_wait_time"
msgid "Process Wait Time"
msgstr ""

36
priv/gettext/errors.pot
View file

@ -110,7 +110,7 @@ msgstr ""
msgid "Can't display this activity"
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:334
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:346
#, elixir-autogen, elixir-format
msgid "Can't find user"
msgstr ""
@ -198,7 +198,7 @@ msgstr ""
msgid "Invalid password."
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:279
#, elixir-autogen, elixir-format
msgid "Invalid request"
msgstr ""
@ -225,7 +225,7 @@ msgstr ""
#: lib/pleroma/web/feed/tag_controller.ex:16
#: lib/pleroma/web/feed/user_controller.ex:69
#: lib/pleroma/web/o_status/o_status_controller.ex:132
#: lib/pleroma/web/plugs/uploaded_media.ex:104
#: lib/pleroma/web/plugs/uploaded_media.ex:84
#, elixir-autogen, elixir-format
msgid "Not found"
msgstr ""
@ -235,7 +235,7 @@ msgstr ""
msgid "Poll's author can't vote"
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:499
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:511
#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20
#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:39
#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:51
@ -341,7 +341,7 @@ msgstr ""
msgid "CAPTCHA expired"
msgstr ""
#: lib/pleroma/web/plugs/uploaded_media.ex:77
#: lib/pleroma/web/plugs/uploaded_media.ex:57
#, elixir-autogen, elixir-format
msgid "Failed"
msgstr ""
@ -361,7 +361,7 @@ msgstr ""
msgid "Insufficient permissions: %{permissions}."
msgstr ""
#: lib/pleroma/web/plugs/uploaded_media.ex:131
#: lib/pleroma/web/plugs/uploaded_media.ex:111
#, elixir-autogen, elixir-format
msgid "Internal Error"
msgstr ""
@ -557,7 +557,7 @@ msgstr ""
msgid "Access denied"
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:331
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:343
#, elixir-autogen, elixir-format
msgid "This API requires an authenticated user"
msgstr ""
@ -567,7 +567,7 @@ msgstr ""
msgid "User is not an admin."
msgstr ""
#: lib/pleroma/user/backup.ex:73
#: lib/pleroma/user/backup.ex:78
#, elixir-format
msgid "Last export was less than a day ago"
msgid_plural "Last export was less than %{days} days ago"
@ -607,3 +607,23 @@ msgstr ""
#, elixir-autogen, elixir-format
msgid "User isn't privileged."
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267
#, elixir-autogen, elixir-format
msgid "Bio is too long"
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:270
#, elixir-autogen, elixir-format
msgid "Name is too long"
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:273
#, elixir-autogen, elixir-format
msgid "One or more field entries are too long"
msgstr ""
#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:276
#, elixir-autogen, elixir-format
msgid "Too many field entries"
msgstr ""

40
priv/gettext/oauth_scopes.pot
View file

@ -219,3 +219,43 @@ msgstr ""
#, elixir-autogen, elixir-format
msgid "read:mutes"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "push"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "read:backups"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "read:chats"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "read:media"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "read:reports"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "write:chats"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "write:follow"
msgstr ""
#: lib/pleroma/web/api_spec/scopes/translator.ex:5
#, elixir-autogen, elixir-format
msgid "write:reports"
msgstr ""

33
priv/repo/migrations/20240527144418_oban_queues_refactor.exs Normal file
View file

@ -0,0 +1,33 @@
defmodule Pleroma.Repo.Migrations.ObanQueuesRefactor do
use Ecto.Migration
@changed_queues [
{"attachments_cleanup", "slow"},
{"check_domain_resolve", "slow"},
{"mailer", "background"},
{"mute_expire", "background"},
{"poll_notifications", "background"},
{"activity_expiration", "slow"},
{"filter_expiration", "background"},
{"token_expiration", "background"},
{"remote_fetcher", "background"},
{"rich_media_expiration", "background"}
]
def up do
Enum.each(@changed_queues, fn {old, new} ->
execute("UPDATE oban_jobs SET queue = '#{new}' WHERE queue = '#{old}';")
end)
# Handled special as reverting this would not be ideal and leaving it is harmless
execute(
"UPDATE oban_jobs SET queue = 'federator_outgoing' WHERE queue = 'scheduled_activities';"
)
end
def down do
# Just move all slow queue jobs to background queue if we are reverting
# as the slow queue will not be processing jobs
execute("UPDATE oban_jobs SET queue = 'background' WHERE queue = 'slow';")
end
end

9
supplemental/search/fastembed-api/Dockerfile Normal file
View file

@ -0,0 +1,9 @@
FROM python:3.9
WORKDIR /code
COPY fastembed-server.py /workdir/fastembed-server.py
COPY requirements.txt /workdir/requirements.txt
RUN pip install -r /workdir/requirements.txt
CMD ["python", "/workdir/fastembed-server.py"]

6
supplemental/search/fastembed-api/README.md Normal file
View file

@ -0,0 +1,6 @@
# About
This is a minimal implementation of the [OpenAI Embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings) meant to be used with the QdrantSearch backend.
# Usage
The easiest way to run it is to just use docker compose with `docker compose up`. This starts the server on the default configured port. Different models can be used, for a full list of supported models, check the [fastembed documentation](https://qdrant.github.io/fastembed/examples/Supported_Models/). The first time a model is requested it will be downloaded, which can take a few seconds.

5
supplemental/search/fastembed-api/compose.yml Normal file
View file

@ -0,0 +1,5 @@
services:
web:
build: .
ports:
- "11345:11345"

27
supplemental/search/fastembed-api/fastembed-server.py Normal file
View file

@ -0,0 +1,27 @@
from fastembed import TextEmbedding
from fastapi import FastAPI
from pydantic import BaseModel
models = {}
app = FastAPI()
class EmbeddingRequest(BaseModel):
model: str
input: str
@app.post("/v1/embeddings")
def embeddings(request: EmbeddingRequest):
model = models.get(request.model) or TextEmbedding(request.model)
models[request.model] = model
embeddings = next(model.embed(request.input)).tolist()
return {"data": [{"embedding": embeddings}]}
@app.get("/health")
def health():
return {"status": "ok"}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=11345)

4
supplemental/search/fastembed-api/requirements.txt Normal file
View file

@ -0,0 +1,4 @@
fastapi==0.111.0
fastembed==0.2.7
pydantic==1.10.15
uvicorn==0.29.0

3
test/pleroma/scheduled_activity_test.exs
View file

@ -31,8 +31,7 @@ test "scheduled activities with jobs when ScheduledActivity enabled" do
{:ok, sa1} = ScheduledActivity.create(user, attrs)
{:ok, sa2} = ScheduledActivity.create(user, attrs)
jobs =
Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args))
jobs = Repo.all(from(j in Oban.Job, where: j.queue == "federator_outgoing", select: j.args))
assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}]
end

199
test/pleroma/search/qdrant_search_test.exs Normal file
View file

@ -0,0 +1,199 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.QdrantSearchTest do
use Pleroma.DataCase, async: true
use Oban.Testing, repo: Pleroma.Repo
import Pleroma.Factory
import Mox
alias Pleroma.Search.QdrantSearch
alias Pleroma.UnstubbedConfigMock, as: Config
alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.SearchIndexingWorker
describe "Qdrant search" do
test "returns the correct healthcheck endpoints" do
# No openai healthcheck URL
Config
|> expect(:get, 2, fn
[Pleroma.Search.QdrantSearch, key], nil ->
%{qdrant_url: "https://qdrant.url"}[key]
end)
[health_endpoint] = QdrantSearch.healthcheck_endpoints()
assert "https://qdrant.url/healthz" == health_endpoint
# Set openai healthcheck URL
Config
|> expect(:get, 2, fn
[Pleroma.Search.QdrantSearch, key], nil ->
%{qdrant_url: "https://qdrant.url", openai_healthcheck_url: "https://openai.url/health"}[
key
]
end)
[_, health_endpoint] = QdrantSearch.healthcheck_endpoints()
assert "https://openai.url/health" == health_endpoint
end
test "searches for a term by encoding it and sending it to qdrant" do
user = insert(:user)
{:ok, activity} =
CommonAPI.post(user, %{
status: "guys i just don't wanna leave the swamp",
visibility: "public"
})
Config
|> expect(:get, 3, fn
[Pleroma.Search, :module], nil ->
QdrantSearch
[Pleroma.Search.QdrantSearch, key], nil ->
%{
openai_model: "a_model",
openai_url: "https://openai.url",
qdrant_url: "https://qdrant.url"
}[key]
end)
Tesla.Mock.mock(fn
%{url: "https://openai.url/v1/embeddings", method: :post} ->
Tesla.Mock.json(%{
data: [%{embedding: [1, 2, 3]}]
})
%{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} ->
data = Jason.decode!(body)
refute data["filter"]
Tesla.Mock.json(%{
result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}]
})
end)
results = QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{})
assert results == [activity]
end
test "for a given actor, ask for only relevant matches" do
user = insert(:user)
{:ok, activity} =
CommonAPI.post(user, %{
status: "guys i just don't wanna leave the swamp",
visibility: "public"
})
Config
|> expect(:get, 3, fn
[Pleroma.Search, :module], nil ->
QdrantSearch
[Pleroma.Search.QdrantSearch, key], nil ->
%{
openai_model: "a_model",
openai_url: "https://openai.url",
qdrant_url: "https://qdrant.url"
}[key]
end)
Tesla.Mock.mock(fn
%{url: "https://openai.url/v1/embeddings", method: :post} ->
Tesla.Mock.json(%{
data: [%{embedding: [1, 2, 3]}]
})
%{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} ->
data = Jason.decode!(body)
assert data["filter"] == %{
"must" => [%{"key" => "actor", "match" => %{"value" => user.ap_id}}]
}
Tesla.Mock.json(%{
result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}]
})
end)
results =
QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{author: user})
assert results == [activity]
end
test "indexes a public post on creation, deletes from the index on deletion" do
user = insert(:user)
Tesla.Mock.mock(fn
%{method: :post, url: "https://openai.url/v1/embeddings"} ->
send(self(), "posted_to_openai")
Tesla.Mock.json(%{
data: [%{embedding: [1, 2, 3]}]
})
%{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} ->
send(self(), "posted_to_qdrant")
data = Jason.decode!(body)
%{"points" => [%{"vector" => vector, "payload" => payload}]} = data
assert vector == [1, 2, 3]
assert payload["actor"]
assert payload["published_at"]
Tesla.Mock.json("ok")
%{method: :post, url: "https://qdrant.url/collections/posts/points/delete"} ->
send(self(), "deleted_from_qdrant")
Tesla.Mock.json("ok")
end)
Config
|> expect(:get, 6, fn
[Pleroma.Search, :module], nil ->
QdrantSearch
[Pleroma.Search.QdrantSearch, key], nil ->
%{
openai_model: "a_model",
openai_url: "https://openai.url",
qdrant_url: "https://qdrant.url"
}[key]
end)
{:ok, activity} =
CommonAPI.post(user, %{
status: "guys i just don't wanna leave the swamp",
visibility: "public"
})
args = %{"op" => "add_to_index", "activity" => activity.id}
assert_enqueued(
worker: SearchIndexingWorker,
args: args
)
assert :ok = perform_job(SearchIndexingWorker, args)
assert_received("posted_to_openai")
assert_received("posted_to_qdrant")
{:ok, _} = CommonAPI.delete(activity.id, user)
delete_args = %{"op" => "remove_from_index", "object" => activity.object.id}
assert_enqueued(worker: SearchIndexingWorker, args: delete_args)
assert :ok = perform_job(SearchIndexingWorker, delete_args)
assert_received("deleted_from_qdrant")
end
end
end

11
test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
View file

@ -27,19 +27,22 @@ test "fails without url" do
end
test "works with honkerific attachments" do
attachment = %{
honk = %{
"mediaType" => "",
"name" => "",
"summary" => "298p3RG7j27tfsZ9RQ.jpg",
"summary" => "Select your spirit chonk",
"name" => "298p3RG7j27tfsZ9RQ.jpg",
"type" => "Document",
"url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg"
}
assert {:ok, attachment} =
AttachmentValidator.cast_and_validate(attachment)
honk
|> AttachmentValidator.cast_and_validate()
|> Ecto.Changeset.apply_action(:insert)
assert attachment.mediaType == "application/octet-stream"
assert attachment.summary == "Select your spirit chonk"
assert attachment.name == "298p3RG7j27tfsZ9RQ.jpg"
end
test "works with an unknown but valid mime type" do

17
test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs
View file

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do
use Oban.Testing, repo: Pleroma.Repo
use Pleroma.Web.ConnCase, async: true
alias Pleroma.Repo
@ -78,7 +79,7 @@ test "updates a scheduled activity" do
}
)
job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities"))
job = Repo.one(from(j in Oban.Job, where: j.queue == "federator_outgoing"))
assert job.args == %{"activity_id" => scheduled_activity.id}
assert DateTime.truncate(job.scheduled_at, :second) == to_datetime(scheduled_at)
@ -124,9 +125,11 @@ test "deletes a scheduled activity" do
}
)
job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities"))
assert job.args == %{"activity_id" => scheduled_activity.id}
assert_enqueued(
worker: Pleroma.Workers.ScheduledActivityWorker,
args: %{"activity_id" => scheduled_activity.id},
queue: :federator_outgoing
)
res_conn =
conn
@ -135,7 +138,11 @@ test "deletes a scheduled activity" do
assert %{} = json_response_and_validate_schema(res_conn, 200)
refute Repo.get(ScheduledActivity, scheduled_activity.id)
refute Repo.get(Oban.Job, job.id)
refute_enqueued(
worker: Pleroma.Workers.ScheduledActivityWorker,
args: %{"activity_id" => scheduled_activity.id}
)
res_conn =
conn

101
test/pleroma/web/mastodon_api/views/status_view_test.exs
View file

@ -593,45 +593,78 @@ test "create mentions from the 'tag' field" do
assert mention.url == recipient.ap_id
end
test "attachments" do
object = %{
"type" => "Image",
"url" => [
%{
"mediaType" => "image/png",
"href" => "someurl",
"width" => 200,
"height" => 100
}
],
"blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn",
"uuid" => 6
}
describe "attachments" do
test "Complete Mastodon style" do
object = %{
"type" => "Image",
"url" => [
%{
"mediaType" => "image/png",
"href" => "someurl",
"width" => 200,
"height" => 100
}
],
"blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn",
"uuid" => 6
}
expected = %{
id: "1638338801",
type: "image",
url: "someurl",
remote_url: "someurl",
preview_url: "someurl",
text_url: "someurl",
description: nil,
pleroma: %{mime_type: "image/png"},
meta: %{original: %{width: 200, height: 100, aspect: 2}},
blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn"
}
expected = %{
id: "1638338801",
type: "image",
url: "someurl",
remote_url: "someurl",
preview_url: "someurl",
text_url: "someurl",
description: nil,
pleroma: %{mime_type: "image/png"},
meta: %{original: %{width: 200, height: 100, aspect: 2}},
blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn"
}
api_spec = Pleroma.Web.ApiSpec.spec()
api_spec = Pleroma.Web.ApiSpec.spec()
assert expected == StatusView.render("attachment.json", %{attachment: object})
assert_schema(expected, "Attachment", api_spec)
assert expected == StatusView.render("attachment.json", %{attachment: object})
assert_schema(expected, "Attachment", api_spec)
# If theres a "id", use that instead of the generated one
object = Map.put(object, "id", 2)
result = StatusView.render("attachment.json", %{attachment: object})
# If theres a "id", use that instead of the generated one
object = Map.put(object, "id", 2)
result = StatusView.render("attachment.json", %{attachment: object})
assert %{id: "2"} = result
assert_schema(result, "Attachment", api_spec)
assert %{id: "2"} = result
assert_schema(result, "Attachment", api_spec)
end
test "Honkerific" do
object = %{
"type" => "Image",
"url" => [
%{
"mediaType" => "image/png",
"href" => "someurl"
}
],
"name" => "fool.jpeg",
"summary" => "they have played us for absolute fools."
}
expected = %{
blurhash: nil,
description: "they have played us for absolute fools.",
id: "1638338801",
pleroma: %{mime_type: "image/png", name: "fool.jpeg"},
preview_url: "someurl",
remote_url: "someurl",
text_url: "someurl",
type: "image",
url: "someurl"
}
api_spec = Pleroma.Web.ApiSpec.spec()
assert expected == StatusView.render("attachment.json", %{attachment: object})
assert_schema(expected, "Attachment", api_spec)
end
end
test "put the url advertised in the Activity in to the url attribute" do

208
test/pleroma/web/plugs/http_security_plug_test.exs
View file

@ -3,14 +3,52 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
use Pleroma.Web.ConnCase
use Pleroma.Web.ConnCase, async: true
alias Plug.Conn
describe "http security enabled" do
setup do: clear_config([:http_security, :enabled], true)
import Mox
test "it sends CSP headers when enabled", %{conn: conn} do
setup do
base_config = Pleroma.Config.get([:http_security])
%{base_config: base_config}
end
defp mock_config(config, additional \\ %{}) do
Pleroma.StaticStubbedConfigMock
|> stub(:get, fn
[:http_security, key] -> config[key]
key -> additional[key]
end)
end
describe "http security enabled" do
setup %{base_config: base_config} do
%{base_config: Keyword.put(base_config, :enabled, true)}
end
test "it does not contain unsafe-eval", %{conn: conn, base_config: base_config} do
mock_config(base_config)
conn = get(conn, "/api/v1/instance")
[header] = Conn.get_resp_header(conn, "content-security-policy")
refute header =~ ~r/unsafe-eval/
end
test "with allow_unsafe_eval set, it does contain it", %{conn: conn, base_config: base_config} do
base_config =
base_config
|> Keyword.put(:allow_unsafe_eval, true)
mock_config(base_config)
conn = get(conn, "/api/v1/instance")
[header] = Conn.get_resp_header(conn, "content-security-policy")
assert header =~ ~r/unsafe-eval/
end
test "it sends CSP headers when enabled", %{conn: conn, base_config: base_config} do
mock_config(base_config)
conn = get(conn, "/api/v1/instance")
refute Conn.get_resp_header(conn, "x-xss-protection") == []
@ -22,8 +60,10 @@ test "it sends CSP headers when enabled", %{conn: conn} do
refute Conn.get_resp_header(conn, "content-security-policy") == []
end
test "it sends STS headers when enabled", %{conn: conn} do
clear_config([:http_security, :sts], true)
test "it sends STS headers when enabled", %{conn: conn, base_config: base_config} do
base_config
|> Keyword.put(:sts, true)
|> mock_config()
conn = get(conn, "/api/v1/instance")
@ -31,8 +71,10 @@ test "it sends STS headers when enabled", %{conn: conn} do
refute Conn.get_resp_header(conn, "expect-ct") == []
end
test "it does not send STS headers when disabled", %{conn: conn} do
clear_config([:http_security, :sts], false)
test "it does not send STS headers when disabled", %{conn: conn, base_config: base_config} do
base_config
|> Keyword.put(:sts, false)
|> mock_config()
conn = get(conn, "/api/v1/instance")
@ -40,19 +82,30 @@ test "it does not send STS headers when disabled", %{conn: conn} do
assert Conn.get_resp_header(conn, "expect-ct") == []
end
test "referrer-policy header reflects configured value", %{conn: conn} do
resp = get(conn, "/api/v1/instance")
test "referrer-policy header reflects configured value", %{
conn: conn,
base_config: base_config
} do
mock_config(base_config)
resp = get(conn, "/api/v1/instance")
assert Conn.get_resp_header(resp, "referrer-policy") == ["same-origin"]
clear_config([:http_security, :referrer_policy], "no-referrer")
base_config
|> Keyword.put(:referrer_policy, "no-referrer")
|> mock_config
resp = get(conn, "/api/v1/instance")
assert Conn.get_resp_header(resp, "referrer-policy") == ["no-referrer"]
end
test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} do
test "it sends `report-to` & `report-uri` CSP response headers", %{
conn: conn,
base_config: base_config
} do
mock_config(base_config)
conn = get(conn, "/api/v1/instance")
[csp] = Conn.get_resp_header(conn, "content-security-policy")
@ -65,7 +118,11 @@ test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} d
"{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}"
end
test "default values for img-src and media-src with disabled media proxy", %{conn: conn} do
test "default values for img-src and media-src with disabled media proxy", %{
conn: conn,
base_config: base_config
} do
mock_config(base_config)
conn = get(conn, "/api/v1/instance")
[csp] = Conn.get_resp_header(conn, "content-security-policy")
@ -73,60 +130,129 @@ test "default values for img-src and media-src with disabled media proxy", %{con
assert csp =~ "img-src 'self' data: blob: https:;"
end
test "it sets the Service-Worker-Allowed header", %{conn: conn} do
clear_config([:http_security, :enabled], true)
clear_config([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"})
test "it sets the Service-Worker-Allowed header", %{conn: conn, base_config: base_config} do
base_config
|> Keyword.put(:enabled, true)
clear_config([:frontends, :available], %{
"fedi-fe" => %{
"name" => "fedi-fe",
"custom-http-headers" => [{"service-worker-allowed", "/"}]
}
})
additional_config =
%{}
|> Map.put([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"})
|> Map.put(
[:frontends, :available],
%{
"fedi-fe" => %{
"name" => "fedi-fe",
"custom-http-headers" => [{"service-worker-allowed", "/"}]
}
}
)
mock_config(base_config, additional_config)
conn = get(conn, "/api/v1/instance")
assert Conn.get_resp_header(conn, "service-worker-allowed") == ["/"]
end
end
describe "img-src and media-src" do
setup do
clear_config([:http_security, :enabled], true)
clear_config([:media_proxy, :enabled], true)
clear_config([:media_proxy, :proxy_opts, :redirect_on_failure], false)
setup %{base_config: base_config} do
base_config =
base_config
|> Keyword.put(:enabled, true)
additional_config =
%{}
|> Map.put([:media_proxy, :enabled], true)
|> Map.put([:media_proxy, :proxy_opts, :redirect_on_failure], false)
|> Map.put([:media_proxy, :whitelist], [])
%{base_config: base_config, additional_config: additional_config}
end
test "media_proxy with base_url", %{conn: conn} do
test "media_proxy with base_url", %{
conn: conn,
base_config: base_config,
additional_config: additional_config
} do
url = "https://example.com"
clear_config([:media_proxy, :base_url], url)
additional_config =
additional_config
|> Map.put([:media_proxy, :base_url], url)
mock_config(base_config, additional_config)
assert_media_img_src(conn, url)
end
test "upload with base url", %{conn: conn} do
test "upload with base url", %{
conn: conn,
base_config: base_config,
additional_config: additional_config
} do
url = "https://example2.com"
clear_config([Pleroma.Upload, :base_url], url)
additional_config =
additional_config
|> Map.put([Pleroma.Upload, :base_url], url)
mock_config(base_config, additional_config)
assert_media_img_src(conn, url)
end
test "with S3 public endpoint", %{conn: conn} do
test "with S3 public endpoint", %{
conn: conn,
base_config: base_config,
additional_config: additional_config
} do
url = "https://example3.com"
clear_config([Pleroma.Uploaders.S3, :public_endpoint], url)
additional_config =
additional_config
|> Map.put([Pleroma.Uploaders.S3, :public_endpoint], url)
mock_config(base_config, additional_config)
assert_media_img_src(conn, url)
end
test "with captcha endpoint", %{conn: conn} do
clear_config([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com")
test "with captcha endpoint", %{
conn: conn,
base_config: base_config,
additional_config: additional_config
} do
additional_config =
additional_config
|> Map.put([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com")
|> Map.put([Pleroma.Captcha, :method], Pleroma.Captcha.Mock)
mock_config(base_config, additional_config)
assert_media_img_src(conn, "https://captcha.com")
end
test "with media_proxy whitelist", %{conn: conn} do
clear_config([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"])
test "with media_proxy whitelist", %{
conn: conn,
base_config: base_config,
additional_config: additional_config
} do
additional_config =
additional_config
|> Map.put([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"])
mock_config(base_config, additional_config)
assert_media_img_src(conn, "https://example7.com https://example6.com")
end
# TODO: delete after removing support bare domains for media proxy whitelist
test "with media_proxy bare domains whitelist (deprecated)", %{conn: conn} do
clear_config([:media_proxy, :whitelist], ["example4.com", "example5.com"])
test "with media_proxy bare domains whitelist (deprecated)", %{
conn: conn,
base_config: base_config,
additional_config: additional_config
} do
additional_config =
additional_config
|> Map.put([:media_proxy, :whitelist], ["example4.com", "example5.com"])
mock_config(base_config, additional_config)
assert_media_img_src(conn, "example5.com example4.com")
end
end
@ -138,8 +264,10 @@ defp assert_media_img_src(conn, url) do
assert csp =~ "img-src 'self' data: blob: #{url};"
end
test "it does not send CSP headers when disabled", %{conn: conn} do
clear_config([:http_security, :enabled], false)
test "it does not send CSP headers when disabled", %{conn: conn, base_config: base_config} do
base_config
|> Keyword.put(:enabled, false)
|> mock_config
conn = get(conn, "/api/v1/instance")

207
test/pleroma/web/plugs/http_signature_plug_test.exs
View file

@ -3,89 +3,89 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
use Pleroma.Web.ConnCase
use Pleroma.Web.ConnCase, async: true
alias Pleroma.StaticStubbedConfigMock, as: ConfigMock
alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock
alias Pleroma.Web.Plugs.HTTPSignaturePlug
import Plug.Conn
import Mox
import Phoenix.Controller, only: [put_format: 2]
import Mock
import Plug.Conn
test "it call HTTPSignatures to check validity if the actor signed it" do
test "it calls HTTPSignatures to check validity if the actor signed it" do
params = %{"actor" => "http://mastodon.example.org/users/admin"}
conn = build_conn(:get, "/doesntmattter", params)
with_mock HTTPSignatures,
validate_conn: fn _ -> true end,
signature_for_conn: fn _ ->
%{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
end do
conn =
conn
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> put_format("activity+json")
|> HTTPSignaturePlug.call(%{})
HTTPSignaturesMock
|> expect(:validate_conn, fn _ -> true end)
assert conn.assigns.valid_signature == true
assert conn.halted == false
assert called(HTTPSignatures.validate_conn(:_))
end
conn =
conn
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> put_format("activity+json")
|> HTTPSignaturePlug.call(%{})
assert conn.assigns.valid_signature == true
assert conn.halted == false
end
describe "requires a signature when `authorized_fetch_mode` is enabled" do
setup do
clear_config([:activitypub, :authorized_fetch_mode], true)
params = %{"actor" => "http://mastodon.example.org/users/admin"}
conn = build_conn(:get, "/doesntmattter", params) |> put_format("activity+json")
[conn: conn]
end
test "when signature header is present", %{conn: conn} do
with_mock HTTPSignatures,
validate_conn: fn _ -> false end,
signature_for_conn: fn _ ->
%{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
end do
conn =
conn
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> HTTPSignaturePlug.call(%{})
test "when signature header is present", %{conn: orig_conn} do
ConfigMock
|> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
|> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end)
assert conn.assigns.valid_signature == false
assert conn.halted == true
assert conn.status == 401
assert conn.state == :sent
assert conn.resp_body == "Request not signed"
assert called(HTTPSignatures.validate_conn(:_))
end
HTTPSignaturesMock
|> expect(:validate_conn, 2, fn _ -> false end)
with_mock HTTPSignatures,
validate_conn: fn _ -> true end,
signature_for_conn: fn _ ->
%{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
end do
conn =
conn
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> HTTPSignaturePlug.call(%{})
conn =
orig_conn
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> HTTPSignaturePlug.call(%{})
assert conn.assigns.valid_signature == true
assert conn.halted == false
assert called(HTTPSignatures.validate_conn(:_))
end
assert conn.assigns.valid_signature == false
assert conn.halted == true
assert conn.status == 401
assert conn.state == :sent
assert conn.resp_body == "Request not signed"
ConfigMock
|> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
HTTPSignaturesMock
|> expect(:validate_conn, fn _ -> true end)
conn =
orig_conn
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> HTTPSignaturePlug.call(%{})
assert conn.assigns.valid_signature == true
assert conn.halted == false
end
test "halts the connection when `signature` header is not present", %{conn: conn} do
ConfigMock
|> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
|> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end)
conn = HTTPSignaturePlug.call(conn, %{})
assert conn.assigns[:valid_signature] == nil
assert conn.halted == true
@ -93,48 +93,73 @@ test "halts the connection when `signature` header is not present", %{conn: conn
assert conn.state == :sent
assert conn.resp_body == "Request not signed"
end
end
test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do
clear_config([:activitypub, :authorized_fetch_mode], true)
clear_config([:instance, :rejected_instances], [{"mastodon.example.org", "no reason"}])
test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do
ConfigMock
|> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
|> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] ->
["192.168.0.0/24"]
end)
|> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
HTTPSignaturesMock
|> expect(:validate_conn, 2, fn _ -> false end)
with_mock HTTPSignatures,
validate_conn: fn _ -> true end,
signature_for_conn: fn _ ->
%{"keyId" => "http://mastodon.example.org/users/admin#main-key"}
end do
conn =
build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
conn
|> Map.put(:remote_ip, {192, 168, 0, 1})
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> put_format("activity+json")
|> HTTPSignaturePlug.call(%{})
assert conn.assigns.valid_signature == true
assert conn.halted == true
assert called(HTTPSignatures.validate_conn(:_))
end
with_mock HTTPSignatures,
validate_conn: fn _ -> true end,
signature_for_conn: fn _ ->
%{"keyId" => "http://allowed.example.org/users/admin#main-key"}
end do
conn =
build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"})
|> put_req_header(
"signature",
"keyId=\"http://allowed.example.org/users/admin#main-key"
)
|> put_format("activity+json")
|> HTTPSignaturePlug.call(%{})
assert conn.assigns.valid_signature == true
assert conn.remote_ip == {192, 168, 0, 1}
assert conn.halted == false
assert called(HTTPSignatures.validate_conn(:_))
end
end
test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do
ConfigMock
|> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
|> expect(:get, fn [:instance, :rejected_instances] ->
[{"mastodon.example.org", "no reason"}]
end)
HTTPSignaturesMock
|> expect(:validate_conn, fn _ -> true end)
conn =
build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
|> put_req_header(
"signature",
"keyId=\"http://mastodon.example.org/users/admin#main-key"
)
|> put_format("activity+json")
|> HTTPSignaturePlug.call(%{})
assert conn.assigns.valid_signature == true
assert conn.halted == true
ConfigMock
|> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end)
|> expect(:get, fn [:instance, :rejected_instances] ->
[{"mastodon.example.org", "no reason"}]
end)
HTTPSignaturesMock
|> expect(:validate_conn, fn _ -> true end)
conn =
build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"})
|> put_req_header(
"signature",
"keyId=\"http://allowed.example.org/users/admin#main-key"
)
|> put_format("activity+json")
|> HTTPSignaturePlug.call(%{})
assert conn.assigns.valid_signature == true
assert conn.halted == false
end
end

1
test/support/data_case.ex
View file

@ -116,6 +116,7 @@ def stub_pipeline do
Mox.stub_with(Pleroma.Web.FederatorMock, Pleroma.Web.Federator)
Mox.stub_with(Pleroma.ConfigMock, Pleroma.Config)
Mox.stub_with(Pleroma.StaticStubbedConfigMock, Pleroma.Test.StaticConfig)
Mox.stub_with(Pleroma.StubbedHTTPSignaturesMock, Pleroma.Test.HTTPSignaturesProxy)
end
def ensure_local_uploader(context) do

9
test/support/http_signatures_proxy.ex Normal file
View file

@ -0,0 +1,9 @@
defmodule Pleroma.Test.HTTPSignaturesProxy do
@behaviour Pleroma.HTTPSignaturesAPI
@impl true
defdelegate validate_conn(conn), to: HTTPSignatures
@impl true
defdelegate signature_for_conn(conn), to: HTTPSignatures
end

1
test/support/mocks.ex
View file

@ -28,6 +28,7 @@
Mox.defmock(Pleroma.ConfigMock, for: Pleroma.Config.Getting)
Mox.defmock(Pleroma.UnstubbedConfigMock, for: Pleroma.Config.Getting)
Mox.defmock(Pleroma.StaticStubbedConfigMock, for: Pleroma.Config.Getting)
Mox.defmock(Pleroma.StubbedHTTPSignaturesMock, for: Pleroma.HTTPSignaturesAPI)
Mox.defmock(Pleroma.LoggerMock, for: Pleroma.Logging)