From c7dc5ce85c5b143df45031d5c87eee0689da3713 Mon Sep 17 00:00:00 2001 From: silverpill Date: Thu, 29 Dec 2022 01:12:32 +0000 Subject: [PATCH 01/35] TagValidator: Allow unrecognized Tag types --- .../web/activity_pub/object_validators/tag_validator.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index 9f15f19815..ad968994eb 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -68,6 +68,9 @@ def changeset(struct, %{"type" => "Emoji"} = data) do |> validate_required([:type, :name, :icon]) end + # Fallback + def changeset(struct, data), do: cast(struct, data, [:type, :name]) + def icon_changeset(struct, data) do struct |> cast(data, [:type, :url]) From 45646ff52cd045485c6be92713edcd0b37225bc3 Mon Sep 17 00:00:00 2001 From: silverpill Date: Thu, 29 Dec 2022 23:57:19 +0000 Subject: [PATCH 02/35] TagValidator: Add test for Link tag --- test/fixtures/fep-e232.json | 26 +++++++++++++++++++ .../web/activity_pub/transmogrifier_test.exs | 16 ++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/fixtures/fep-e232.json diff --git a/test/fixtures/fep-e232.json b/test/fixtures/fep-e232.json new file mode 100644 index 0000000000..98f3395894 --- /dev/null +++ b/test/fixtures/fep-e232.json @@ -0,0 +1,26 @@ +{ + "@context": "https://www.w3.org/ns/activitystreams", + "type": "Create", + "actor": "https://example.org/users/alice", + "object": { + "id": "https://example.org/objects/10", + "type": "Note", + "attributedTo": "https://example.org/users/alice", + "content": "

test https://example.org/objects/9

", + "published": "2022-10-01T21:30:05.211215Z", + "tag": [ + { + "name": "https://example.org/objects/9", + "type": "Link", + "href": "https://example.org/objects/9", + "mediaType": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + } + ], + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://example.org/users/alice/followers" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 6b4636d227..c0425f30cd 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -123,6 +123,22 @@ test "it fixes both the Create and object contexts in a reply" do assert activity.data["context"] == object.data["context"] end + + # https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-e232.md + test "it accepts FEP-e232 link tags" do + insert(:user, ap_id: "https://example.org/users/alice") + + message = File.read!("test/fixtures/fep-e232.json") |> Jason.decode!() + + assert {:ok, activity} = Transmogrifier.handle_incoming(message) + + object = Object.normalize(activity) + assert length(object.data["tag"]) == 1 + + tag = object.data["tag"] |> List.first() + assert tag["type"] == "Link" + assert tag["name"] == "https://example.org/objects/9" + end end describe "prepare outgoing" do From 5cfb0578a6845db377b5679ac05aa25ee5656211 Mon Sep 17 00:00:00 2001 From: silverpill Date: Fri, 30 Dec 2022 17:26:43 +0000 Subject: [PATCH 03/35] TagValidator: Drop unrecognized tags --- .../web/activity_pub/object_validators/tag_validator.ex | 7 +++++-- test/fixtures/fep-e232.json | 5 +++++ test/pleroma/web/activity_pub/transmogrifier_test.exs | 6 ++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index ad968994eb..cfd510c192 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -68,8 +68,11 @@ def changeset(struct, %{"type" => "Emoji"} = data) do |> validate_required([:type, :name, :icon]) end - # Fallback - def changeset(struct, data), do: cast(struct, data, [:type, :name]) + def changeset(struct, %{"type" => _} = data) do + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end def icon_changeset(struct, data) do struct diff --git a/test/fixtures/fep-e232.json b/test/fixtures/fep-e232.json index 98f3395894..e9d12ae357 100644 --- a/test/fixtures/fep-e232.json +++ b/test/fixtures/fep-e232.json @@ -9,6 +9,11 @@ "content": "

test https://example.org/objects/9

", "published": "2022-10-01T21:30:05.211215Z", "tag": [ + { + "name": "@bob@example.net", + "type": "Mention", + "href": "https://example.net/users/bob" + }, { "name": "https://example.org/objects/9", "type": "Link", diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index c0425f30cd..f76606479d 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -124,8 +124,7 @@ test "it fixes both the Create and object contexts in a reply" do assert activity.data["context"] == object.data["context"] end - # https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-e232.md - test "it accepts FEP-e232 link tags" do + test "it drops link tags" do insert(:user, ap_id: "https://example.org/users/alice") message = File.read!("test/fixtures/fep-e232.json") |> Jason.decode!() @@ -136,8 +135,7 @@ test "it accepts FEP-e232 link tags" do assert length(object.data["tag"]) == 1 tag = object.data["tag"] |> List.first() - assert tag["type"] == "Link" - assert tag["name"] == "https://example.org/objects/9" + assert tag["type"] == "Mention" end end From 22b72cd6b87b4a684bde9c34741e5bb1e1204ed0 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Sun, 5 Mar 2023 06:37:59 +0000 Subject: [PATCH 04/35] Merge branch 'tusooa/oban-common-pipeline' into 'develop' Stop oban from retrying if validating errors occur when processing incoming data See merge request pleroma/pleroma!3844 --- lib/pleroma/workers/receiver_worker.ex | 3 ++ test/pleroma/web/federator_test.exs | 2 +- test/pleroma/workers/receiver_worker_test.exs | 29 ++++++++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 4f513b9074..cf1bb62b44 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -13,6 +13,9 @@ def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do {:ok, res} else {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} + {:error, :already_present} -> {:cancel, :already_present} + {:error, {:validate_object, reason}} -> {:cancel, reason} + {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} e -> e end diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs index 41d1c5d5e8..1ffe6aae15 100644 --- a/test/pleroma/web/federator_test.exs +++ b/test/pleroma/web/federator_test.exs @@ -133,7 +133,7 @@ test "successfully processes incoming AP docs with correct origin" do assert {:ok, _activity} = ObanHelpers.perform(job) assert {:ok, job} = Federator.incoming_ap_doc(params) - assert {:error, :already_present} = ObanHelpers.perform(job) + assert {:cancel, :already_present} = ObanHelpers.perform(job) end test "rejects incoming AP docs with incorrect origin" do diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 283beee4d5..acea0ae000 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -11,7 +11,7 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do alias Pleroma.Workers.ReceiverWorker - test "it ignores MRF reject" do + test "it does not retry MRF reject" do params = insert(:note).data with_mock Pleroma.Web.ActivityPub.Transmogrifier, @@ -22,4 +22,31 @@ test "it ignores MRF reject" do }) end end + + test "it does not retry ObjectValidator reject" do + params = + insert(:note_activity).data + |> Map.put("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id()) + |> Map.put("object", %{ + "type" => "Note", + "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id() + }) + + with_mock Pleroma.Web.ActivityPub.ObjectValidator, [:passthrough], + validate: fn _, _ -> {:error, %Ecto.Changeset{}} end do + assert {:cancel, {:error, %Ecto.Changeset{}}} = + ReceiverWorker.perform(%Oban.Job{ + args: %{"op" => "incoming_ap_doc", "params" => params} + }) + end + end + + test "it does not retry duplicates" do + params = insert(:note_activity).data + + assert {:cancel, :already_present} = + ReceiverWorker.perform(%Oban.Job{ + args: %{"op" => "incoming_ap_doc", "params" => params} + }) + end end From d640df39276e19c1adc31d46692e46b7cdcfb991 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Mon, 6 Mar 2023 21:08:27 +0000 Subject: [PATCH 05/35] Merge branch 'fix/static-fe-feed-500' into 'develop' fix: remove static_fe pipeline for /users/:nickname/feed See merge request pleroma/pleroma!3852 --- lib/pleroma/web/router.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ba1d64ab23..c1a690e28a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -835,8 +835,7 @@ defmodule Pleroma.Web.Router do end scope "/", Pleroma.Web do - # Note: html format is supported only if static FE is enabled - pipe_through([:accepts_html_xml, :static_fe]) + pipe_through([:accepts_html_xml]) get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed) end From 937df7e46536fee9703285e8a136bba67f1f907b Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Mon, 6 Mar 2023 22:55:24 +0000 Subject: [PATCH 06/35] Merge branch 'fix/tag-feed-crashes' into 'develop' fix: atom/rss feed issues Closes #3045 See merge request pleroma/pleroma!3851 --- lib/pleroma/web/feed/feed_view.ex | 10 ++--- lib/pleroma/web/metadata/utils.ex | 5 ++- .../templates/feed/feed/_activity.atom.eex | 4 +- .../web/templates/feed/feed/_activity.rss.eex | 2 +- .../feed/feed/_tag_activity.atom.eex | 4 +- .../templates/feed/feed/_tag_activity.xml.eex | 2 +- .../pleroma/web/feed/user_controller_test.exs | 44 +++++++++++++++++-- test/pleroma/web/metadata/utils_test.exs | 2 +- 8 files changed, 56 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 449659f4bb..034722eb2e 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Feed.FeedView do use Phoenix.HTML use Pleroma.Web, :view - alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.Gettext @@ -72,7 +71,9 @@ def logo(user) do def last_activity(activities), do: List.last(activities) - def activity_title(%{"content" => content, "summary" => summary} = data, opts \\ %{}) do + def activity_title(%{"content" => content} = data, opts \\ %{}) do + summary = Map.get(data, "summary", "") + title = cond do summary != "" -> summary @@ -81,9 +82,8 @@ def activity_title(%{"content" => content, "summary" => summary} = data, opts \\ end title - |> Pleroma.Web.Metadata.Utils.scrub_html() - |> Pleroma.Emoji.Formatter.demojify() - |> Formatter.truncate(opts[:max_length], opts[:omission]) + |> Pleroma.Web.Metadata.Utils.scrub_html_and_truncate(opts[:max_length], opts[:omission]) + |> HtmlEntities.encode() end def activity_description(data) do diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 15414a988d..80a8be9a2d 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -30,12 +30,13 @@ def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do |> scrub_html_and_truncate_object_field(object) end - def scrub_html_and_truncate(content, max_length \\ 200) when is_binary(content) do + def scrub_html_and_truncate(content, max_length \\ 200, omission \\ "...") + when is_binary(content) do content |> scrub_html |> Emoji.Formatter.demojify() |> HtmlEntities.decode() - |> Formatter.truncate(max_length) + |> Formatter.truncate(max_length, omission) end def scrub_html(content) when is_binary(content) do diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 260338772a..b774f79844 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -4,8 +4,8 @@ <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= to_rfc3339(@activity.data["published"]) %> - <%= to_rfc3339(@activity.data["published"]) %> + <%= to_rfc3339(@data["published"]) %> + <%= to_rfc3339(@data["published"]) %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 5c8f35fe47..7de98f7368 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -4,7 +4,7 @@ <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= to_rfc2822(@activity.data["published"]) %> + <%= to_rfc2822(@data["published"]) %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 25980c1e42..03c222975e 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -7,8 +7,8 @@ <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= to_rfc3339(@activity.data["published"]) %> - <%= to_rfc3339(@activity.data["published"]) %> + <%= to_rfc3339(@data["published"]) %> + <%= to_rfc3339(@data["published"]) %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index d582c83e8c..1b8c34b871 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -4,7 +4,7 @@ <%= activity_context(@activity) %> <%= activity_context(@activity) %> - <%= to_rfc2822(@activity.data["published"]) %> + <%= to_rfc2822(@data["published"]) %> <%= activity_description(@data) %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/test/pleroma/web/feed/user_controller_test.exs b/test/pleroma/web/feed/user_controller_test.exs index de32d3d4bf..d3c4108de0 100644 --- a/test/pleroma/web/feed/user_controller_test.exs +++ b/test/pleroma/web/feed/user_controller_test.exs @@ -57,9 +57,23 @@ defmodule Pleroma.Web.Feed.UserControllerTest do ) note_activity2 = insert(:note_activity, note: note2) + + note3 = + insert(:note, + user: user, + data: %{ + "content" => "This note tests whether HTML entities are truncated properly", + "summary" => "Won't, didn't fail", + "inReplyTo" => note_activity2.id + } + ) + + _note_activity3 = insert(:note_activity, note: note3) object = Object.normalize(note_activity, fetch: false) - [user: user, object: object, max_id: note_activity2.id] + encoded_title = FeedView.activity_title(note3.data) + + [user: user, object: object, max_id: note_activity2.id, encoded_title: encoded_title] end test "gets an atom feed", %{conn: conn, user: user, object: object, max_id: max_id} do @@ -74,7 +88,7 @@ test "gets an atom feed", %{conn: conn, user: user, object: object, max_id: max_ |> SweetXml.parse() |> SweetXml.xpath(~x"//entry/title/text()"l) - assert activity_titles == ['2hu', '2hu & as'] + assert activity_titles == ['Won\'t, didn\'...', '2hu', '2hu & as'] assert resp =~ FeedView.escape(object.data["content"]) assert resp =~ FeedView.escape(object.data["summary"]) assert resp =~ FeedView.escape(object.data["context"]) @@ -105,7 +119,7 @@ test "gets a rss feed", %{conn: conn, user: user, object: object, max_id: max_id |> SweetXml.parse() |> SweetXml.xpath(~x"//item/title/text()"l) - assert activity_titles == ['2hu', '2hu & as'] + assert activity_titles == ['Won\'t, didn\'...', '2hu', '2hu & as'] assert resp =~ FeedView.escape(object.data["content"]) assert resp =~ FeedView.escape(object.data["summary"]) assert resp =~ FeedView.escape(object.data["context"]) @@ -176,6 +190,30 @@ test "does not require authentication on non-federating instances", %{conn: conn |> get("/users/#{user.nickname}/feed.rss") |> response(200) end + + test "does not mangle HTML entities midway", %{ + conn: conn, + user: user, + object: object, + encoded_title: encoded_title + } do + resp = + conn + |> put_req_header("accept", "application/atom+xml") + |> get(user_feed_path(conn, :feed, user.nickname)) + |> response(200) + + activity_titles = + resp + |> SweetXml.parse() + |> SweetXml.xpath(~x"//entry/title/text()"l) + + assert activity_titles == ['Won\'t, didn\'...', '2hu', '2hu & as'] + assert resp =~ FeedView.escape(object.data["content"]) + assert resp =~ FeedView.escape(object.data["summary"]) + assert resp =~ FeedView.escape(object.data["context"]) + assert resp =~ encoded_title + end end # Note: see ActivityPubControllerTest for JSON format tests diff --git a/test/pleroma/web/metadata/utils_test.exs b/test/pleroma/web/metadata/utils_test.exs index 85ef6033a7..3daf852fba 100644 --- a/test/pleroma/web/metadata/utils_test.exs +++ b/test/pleroma/web/metadata/utils_test.exs @@ -72,7 +72,7 @@ test "it does not return old content after editing" do end end - describe "scrub_html_and_truncate/2" do + describe "scrub_html_and_truncate/3" do test "it returns text without encode HTML" do assert Utils.scrub_html_and_truncate("Pleroma's really cool!") == "Pleroma's really cool!" end From 40f14fd31cb1a92b7f1b2a533a725c709fefbb3b Mon Sep 17 00:00:00 2001 From: tusooa Date: Sun, 12 Mar 2023 16:06:00 +0000 Subject: [PATCH 07/35] Merge branch 'remove-crypt' into 'develop' Remove crypt(3) support Closes #3030 and #3062 See merge request pleroma/pleroma!3847 --- CHANGELOG.md | 1 + lib/pleroma/web/plugs/authentication_plug.ex | 8 ----- mix.exs | 1 - mix.lock | 1 - .../web/plugs/authentication_plug_test.exs | 30 ------------------- 5 files changed, 1 insertion(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a7ec10328..1742ee2f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed ### Removed +- BREAKING: Support for passwords generated with `crypt(3)` (Gnu Social migration artifact) ## 2.5.1 diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex index a7fd697b5d..f912a1542f 100644 --- a/lib/pleroma/web/plugs/authentication_plug.ex +++ b/lib/pleroma/web/plugs/authentication_plug.ex @@ -38,10 +38,6 @@ def call( def call(conn, _), do: conn - def checkpw(password, "$6" <> _ = password_hash) do - :crypt.crypt(password, password_hash) == password_hash - end - def checkpw(password, "$2" <> _ = password_hash) do # Handle bcrypt passwords for Mastodon migration Bcrypt.verify_pass(password, password_hash) @@ -60,10 +56,6 @@ def maybe_update_password(%User{password_hash: "$2" <> _} = user, password) do do_update_password(user, password) end - def maybe_update_password(%User{password_hash: "$6" <> _} = user, password) do - do_update_password(user, password) - end - def maybe_update_password(user, _), do: {:ok, user} defp do_update_password(user, password) do diff --git a/mix.exs b/mix.exs index ab0be4deb0..7439bbca65 100644 --- a/mix.exs +++ b/mix.exs @@ -150,7 +150,6 @@ defp deps do {:sweet_xml, "~> 0.7.2"}, {:earmark, "~> 1.4.22"}, {:bbcode_pleroma, "~> 0.2.0"}, - {:crypt, "~> 1.0"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, {:swoosh, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 3027863262..8419dc7397 100644 --- a/mix.lock +++ b/mix.lock @@ -21,7 +21,6 @@ "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, "credo": {:hex, :credo, "1.6.7", "323f5734350fd23a456f2688b9430e7d517afb313fbd38671b8a4449798a7854", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "41e110bfb007f7eda7f897c10bf019ceab9a0b269ce79f015d54b0dcf4fc7dd3"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, - "crypt": {:hex, :crypt, "1.0.1", "a3567e1c651a2ec42c6650d9f3ab789e0f12a508c060653a9bbb5fafe60f043c", [:rebar3], [], "hexpm", "968dffe321c7a5d9f9b4577c4a4ff56a1c26d1a8a2270eb22c7636a0b43d3982"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.4.2", "f92e79aff2375299a16bcb069a14ee8615c3414863a6fef93156aee8e86c2ff3", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4fe53ca91b99f55ea249693a0229356a08f4d1a7931d8ffa79289b145fe83668"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, diff --git a/test/pleroma/web/plugs/authentication_plug_test.exs b/test/pleroma/web/plugs/authentication_plug_test.exs index 41fdb93bc9..b8acd01c59 100644 --- a/test/pleroma/web/plugs/authentication_plug_test.exs +++ b/test/pleroma/web/plugs/authentication_plug_test.exs @@ -70,28 +70,6 @@ test "with a bcrypt hash, it updates to a pkbdf2 hash", %{conn: conn} do assert "$pbkdf2" <> _ = user.password_hash end - @tag :skip_on_mac - test "with a crypt hash, it updates to a pkbdf2 hash", %{conn: conn} do - user = - insert(:user, - password_hash: - "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - ) - - conn = - conn - |> assign(:auth_user, user) - |> assign(:auth_credentials, %{password: "password"}) - |> AuthenticationPlug.call(%{}) - - assert conn.assigns.user.id == conn.assigns.auth_user.id - assert conn.assigns.token == nil - assert PlugHelper.plug_skipped?(conn, OAuthScopesPlug) - - user = User.get_by_id(user.id) - assert "$pbkdf2" <> _ = user.password_hash - end - describe "checkpw/2" do test "check pbkdf2 hash" do hash = @@ -101,14 +79,6 @@ test "check pbkdf2 hash" do refute AuthenticationPlug.checkpw("test-password1", hash) end - @tag :skip_on_mac - test "check sha512-crypt hash" do - hash = - "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - - assert AuthenticationPlug.checkpw("password", hash) - end - test "check bcrypt hash" do hash = "$2a$10$uyhC/R/zoE1ndwwCtMusK.TLVzkQ/Ugsbqp3uXI.CTTz0gBw.24jS" From ad38cc3b0c7895554291e874dfcfcf011b21f886 Mon Sep 17 00:00:00 2001 From: tusooa Date: Thu, 16 Mar 2023 08:11:44 +0000 Subject: [PATCH 08/35] Merge branch 'docs-otp-support' into 'develop' docs: Be more explicit about the level of compatibility of OTP releases See merge request pleroma/pleroma!3849 --- docs/installation/otp_en.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 8c02201e67..f2812346b1 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -2,15 +2,16 @@ {! backend/installation/otp_vs_from_source.include !} -This guide covers a installation using an OTP release. To install Pleroma from source, please check out the corresponding guide for your distro. +This guide covers a installation using OTP releases as built by the Pleroma project, it is meant as a fallback to distribution packages/recipes which are the preferred installation method. +To install Pleroma from source, please check out the corresponding guide for your distro. ## Pre-requisites -* A machine running Linux with GNU (e.g. Debian, Ubuntu) or musl (e.g. Alpine) libc and `x86_64`, `aarch64` or `armv7l` CPU, you have root access to. If you are not sure if it's compatible see [Detecting flavour section](#detecting-flavour) below +* A machine you have root access to running Debian GNU/Linux or compatible (eg. Ubuntu), or Alpine on `x86_64`, `aarch64` or `armv7l` CPU. If you are not sure what you are running see [Detecting flavour section](#detecting-flavour) below * A (sub)domain pointed to the machine -You will be running commands as root. If you aren't root already, please elevate your privileges by executing `sudo su`/`su`. +You will be running commands as root. If you aren't root already, please elevate your privileges by executing `sudo -i`/`su`. -While in theory OTP releases are possbile to install on any compatible machine, for the sake of simplicity this guide focuses only on Debian/Ubuntu and Alpine. +Similarly to other binaries, OTP releases tend to be only compatible with the distro they are built on, as such this guide focuses only on Debian/Ubuntu and Alpine. ### Detecting flavour @@ -19,7 +20,7 @@ Paste the following into the shell: arch="$(uname -m)";if [ "$arch" = "x86_64" ];then arch="amd64";elif [ "$arch" = "armv7l" ];then arch="arm";elif [ "$arch" = "aarch64" ];then arch="arm64";else echo "Unsupported arch: $arch">&2;fi;if getconf GNU_LIBC_VERSION>/dev/null;then libc_postfix="";elif [ "$(ldd 2>&1|head -c 9)" = "musl libc" ];then libc_postfix="-musl";elif [ "$(find /lib/libc.musl*|wc -l)" ];then libc_postfix="-musl";else echo "Unsupported libc">&2;fi;echo "$arch$libc_postfix" ``` -If your platform is supported the output will contain the flavour string, you will need it later. If not, this just means that we don't build releases for your platform, you can still try installing from source. +This should give your flavour string. If not this just means that we don't build releases for your platform, you can still try installing from source. ### Installing the required packages From e4288df50266a73ae1485c684b18d6f20f3234b0 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Sat, 25 Mar 2023 05:40:35 +0000 Subject: [PATCH 09/35] Merge branch 'background-timeout' into 'develop' Set background worker timeout to 15 minutes See merge request pleroma/pleroma!3857 --- lib/pleroma/workers/background_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 3805293bc3..7944176126 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -45,5 +45,5 @@ def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do end @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) + def timeout(_job), do: :timer.seconds(900) end From 50e237759a0aa378b32393ba57d50eaaff5029e0 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 20:30:17 -0400 Subject: [PATCH 10/35] Use git diff to search for changelog entry --- .gitlab-ci.yml | 1 + tools/check-changelog | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e4897aa0a..6e498bf89d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,6 +49,7 @@ check-changelog: after_script: '' cache: {} script: + - apk add git - sh ./tools/check-changelog build: diff --git a/tools/check-changelog b/tools/check-changelog index b94b52755d..0b57c4d1e4 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -1,22 +1,15 @@ #!/bin/sh -echo "looking for change log of $CI_MERGE_REQUEST_IID" +echo "looking for change log" -count=0 -for i in add remove fix security skip; do - [ -f changelog.d/"$CI_MERGE_REQUEST_IID"."$i" ] - retcode=$? - if [ $retcode -eq 0 ]; then - echo "found $CI_MERGE_REQUEST_IID.$i" - count=$(( count + 1 )) - else - echo "no $CI_MERGE_REQUEST_IID.$i" - fi -done -if [ $count -gt 0 ]; then - echo "ok" +git diff --raw $CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ + grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\)$' +ret=$? + +if [ $ret -eq 0 ]; then + echo "found a changelog entry" exit 0 else - echo "must have a changelog entry or explicitly skip it" + echo "changelog entry not found" exit 1 fi From e13b331762881933d66597b3145b4d20d6e51912 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 20:40:39 -0400 Subject: [PATCH 11/35] Fetch upstream in the repo --- tools/check-changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/check-changelog b/tools/check-changelog index 0b57c4d1e4..1409be9f01 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -2,7 +2,10 @@ echo "looking for change log" -git diff --raw $CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ +git remote add upstream https://git.pleroma.social/pleroma/pleroma.git +git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}:refs/remotes/upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +git diff --raw upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\)$' ret=$? From c1aa83069daaa7df906f63b3417e14b7edbfae94 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 20:45:27 -0400 Subject: [PATCH 12/35] Skip changelog --- changelog.d/changelog-improve.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/changelog-improve.skip diff --git a/changelog.d/changelog-improve.skip b/changelog.d/changelog-improve.skip new file mode 100644 index 0000000000..e69de29bb2 From 30bc37c3cab8ebf7be667fb291f0d99c17d808ef Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 21:02:13 -0400 Subject: [PATCH 13/35] Explain changelog.d in merge request templates --- .gitlab/merge_request_templates/Default.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .gitlab/merge_request_templates/Default.md diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md new file mode 100644 index 0000000000..fdf219f996 --- /dev/null +++ b/.gitlab/merge_request_templates/Default.md @@ -0,0 +1,10 @@ +### Checklist +- [ ] Adding a changelog: In the `changelog.d` directory, create a file named `.`. + + `` can be anything, but we recommend using a more or less unique identifier to avoid collisions, such as the branch name. + + `` can be `add`, `remove`, `fix`, `security` or `skip`. `skip` is only used if there is no user-visible change in the MR (for example, only editing comments in the code). Otherwise, choose a type that corresponds to your change. + + In the file, write the changelog entry. For example, if an MR adds group functionality, we can create a file named `group.add` and write `Add group functionality` in it. + + If one changelog entry is not enough, you may add more. But that might mean you can split it into two MRs. Only use more than one changelog entry if you really need to (for example, when one change in the code fix two different bugs, or when refactoring). From ae8f359f22d8d02c2937df0a81052fc3504d4248 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 21:07:18 -0400 Subject: [PATCH 14/35] Skip changelog check for automated MRs --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e498bf89d..2eee3bc688 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,10 @@ check-changelog: stage: check-changelog image: alpine rules: + - if: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'pleroma/pleroma' && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == 'weblate-extract' + when: never + - if: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'pleroma/pleroma' && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == 'weblate' + when: never - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" before_script: '' after_script: '' From 85bdbb102e8fe4dc4a0eb765d8a2a01308161772 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 4 Mar 2023 19:50:47 -0500 Subject: [PATCH 15/35] Add extraction process for oauth scopes --- lib/pleroma/web/api_spec/scopes/compiler.ex | 82 +++++++++++++++++++ lib/pleroma/web/api_spec/scopes/translator.ex | 10 +++ .../web/api_spec/scopes/compiler_test.exs | 56 +++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 lib/pleroma/web/api_spec/scopes/compiler.ex create mode 100644 lib/pleroma/web/api_spec/scopes/translator.ex create mode 100644 test/pleroma/web/api_spec/scopes/compiler_test.exs diff --git a/lib/pleroma/web/api_spec/scopes/compiler.ex b/lib/pleroma/web/api_spec/scopes/compiler.ex new file mode 100644 index 0000000000..162edc9a38 --- /dev/null +++ b/lib/pleroma/web/api_spec/scopes/compiler.ex @@ -0,0 +1,82 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Scopes.Compiler do + defmacro __before_compile__(_env) do + strings = __MODULE__.extract_all_scopes() + + quote do + def placeholder do + unquote do + Enum.map( + strings, + fn string -> + quote do + Pleroma.Web.Gettext.dgettext_noop( + "oauth_scopes", + unquote(string) + ) + end + end + ) + end + end + end + end + + def extract_all_scopes do + extract_all_scopes_from(Pleroma.Web.ApiSpec.spec()) + end + + def extract_all_scopes_from(specs) do + specs.paths + |> Enum.reduce([], fn + {_path, %{} = path_item}, acc -> + extract_routes(path_item) + |> Enum.flat_map(fn operation -> process_operation(operation) end) + |> Kernel.++(acc) + + {_, _}, acc -> + acc + end) + |> Enum.uniq() + end + + defp extract_routes(path_item) do + path_item + |> Map.from_struct() + |> Enum.map(fn {_method, path_item} -> path_item end) + |> Enum.filter(fn + %OpenApiSpex.Operation{} = _operation -> true + _ -> false + end) + end + + defp process_operation(operation) do + operation.security + |> Kernel.||([]) + |> Enum.flat_map(fn + %{"oAuth" => scopes} -> process_scopes(scopes) + _ -> [] + end) + end + + defp process_scopes(scopes) do + scopes + |> Enum.flat_map(fn scope -> + process_scope(scope) + end) + end + + def process_scope(scope) do + hierarchy = String.split(scope, ":") + + {_, list} = + Enum.reduce(hierarchy, {"", []}, fn comp, {cur, list} -> + {cur <> comp <> ":", [cur <> comp | list]} + end) + + list + end +end diff --git a/lib/pleroma/web/api_spec/scopes/translator.ex b/lib/pleroma/web/api_spec/scopes/translator.ex new file mode 100644 index 0000000000..54eea35937 --- /dev/null +++ b/lib/pleroma/web/api_spec/scopes/translator.ex @@ -0,0 +1,10 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Scopes.Translator do + require Pleroma.Web.ApiSpec.Scopes.Compiler + require Pleroma.Web.Gettext + + @before_compile Pleroma.Web.ApiSpec.Scopes.Compiler +end diff --git a/test/pleroma/web/api_spec/scopes/compiler_test.exs b/test/pleroma/web/api_spec/scopes/compiler_test.exs new file mode 100644 index 0000000000..99e1d343a9 --- /dev/null +++ b/test/pleroma/web/api_spec/scopes/compiler_test.exs @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Scopes.CompilerTest do + use ExUnit.Case, async: true + + alias Pleroma.Web.ApiSpec.Scopes.Compiler + + @dummy_response %{} + + @data %{ + paths: %{ + "/mew" => %OpenApiSpex.PathItem{ + post: %OpenApiSpex.Operation{ + security: [%{"oAuth" => ["a:b:c"]}], + responses: @dummy_response + }, + get: %OpenApiSpex.Operation{security: nil, responses: @dummy_response} + }, + "/mew2" => %OpenApiSpex.PathItem{ + post: %OpenApiSpex.Operation{ + security: [%{"oAuth" => ["d:e", "f:g"]}], + responses: @dummy_response + }, + get: %OpenApiSpex.Operation{security: nil, responses: @dummy_response} + } + } + } + + describe "process_scope/1" do + test "gives all higher-level scopes" do + scopes = Compiler.process_scope("admin:read:accounts") + + assert [_, _, _] = scopes + assert "admin" in scopes + assert "admin:read" in scopes + assert "admin:read:accounts" in scopes + end + end + + describe "extract_all_scopes_from/1" do + test "extracts scopes" do + scopes = Compiler.extract_all_scopes_from(@data) + + assert [_, _, _, _, _, _, _] = scopes + assert "a" in scopes + assert "a:b" in scopes + assert "a:b:c" in scopes + assert "d" in scopes + assert "d:e" in scopes + assert "f" in scopes + assert "f:g" in scopes + end + end +end From 530284e1b96b6daeb21995079ff21544e545aa7f Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 4 Mar 2023 19:57:42 -0500 Subject: [PATCH 16/35] Add extracted pot --- priv/gettext/oauth_scopes.pot | 261 ++++++++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 priv/gettext/oauth_scopes.pot diff --git a/priv/gettext/oauth_scopes.pot b/priv/gettext/oauth_scopes.pot new file mode 100644 index 0000000000..5f7b425f3e --- /dev/null +++ b/priv/gettext/oauth_scopes.pot @@ -0,0 +1,261 @@ +## This file is a PO Template file. +## +## "msgid"s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run "mix gettext.extract" to bring this file up to +## date. Leave "msgstr"s empty as changing them here has no +## effect: edit them in PO (.po) files instead. +msgid "" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "follow" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "push" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:backups" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:blocks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:bookmarks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:favourites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:filters" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:follows" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:lists" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:notifications" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:search" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:blocks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:bookmarks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:conversations" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:favourites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:filters" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follow" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follows" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:lists" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:media" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:mutes" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:notifications" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:invites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:media_proxy_caches" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:follows" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:invites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:media_proxy_caches" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:media" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:mutes" +msgstr "" From 6d0ebccdb03f6d19bccb01a6c3ae4e7f647aa678 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 4 Mar 2023 19:58:05 -0500 Subject: [PATCH 17/35] Make webui use translated scope descriptions --- lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex index 73115e92a2..7585c4d3ee 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex @@ -8,7 +8,7 @@ <%= checkbox @form, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> <%= label @form, :"scope_#{scope}", String.capitalize(scope) %> <%= if scope in @scopes && scope do %> - <%= String.capitalize(scope) %> + <%= scope %> <%= :"Elixir.Gettext".dgettext(Gettext, "oauth_scopes", scope) %> <% end %> <% else %> From b6dd194000a4a9ce23dc844758fa797f08a01550 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 16:33:53 -0400 Subject: [PATCH 18/35] Add changelog --- changelog.d/3848.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3848.add diff --git a/changelog.d/3848.add b/changelog.d/3848.add new file mode 100644 index 0000000000..d7b1b0a841 --- /dev/null +++ b/changelog.d/3848.add @@ -0,0 +1 @@ +Add OAuth scope descriptions From 9283c784a38194a937dc1a46a6b46478d159e434 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 17:03:09 -0400 Subject: [PATCH 19/35] Add English translation for oauth scopes --- priv/gettext/en/LC_MESSAGES/oauth_scopes.po | 264 ++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 priv/gettext/en/LC_MESSAGES/oauth_scopes.po diff --git a/priv/gettext/en/LC_MESSAGES/oauth_scopes.po b/priv/gettext/en/LC_MESSAGES/oauth_scopes.po new file mode 100644 index 0000000000..105ca022bb --- /dev/null +++ b/priv/gettext/en/LC_MESSAGES/oauth_scopes.po @@ -0,0 +1,264 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR Free Software Foundation, Inc. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"PO-Revision-Date: 2023-05-02 17:02-0400\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: 8bit\n" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin" +msgstr "All admin access" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read" +msgstr "Read all using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write" +msgstr "Write all using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "follow" +msgstr "Read and write user relationships" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "push" +msgstr "Push notifications" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read" +msgstr "Read everything" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:accounts" +msgstr "Read information of all accounts" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:backups" +msgstr "Read your backups" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:blocks" +msgstr "Read block relationships" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:bookmarks" +msgstr "Read your bookmarks" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:chats" +msgstr "Read your chats" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:favourites" +msgstr "Read your favourites" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:filters" +msgstr "Read your filtering settings" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:follows" +msgstr "Read follow relationships" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:lists" +msgstr "Read your lists" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:notifications" +msgstr "Read your notifications" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:reports" +msgstr "Read your reports" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:search" +msgstr "Perform searches" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:statuses" +msgstr "Read all statuses you can see" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write" +msgstr "Write everything" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:accounts" +msgstr "Change your account information" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:blocks" +msgstr "Block or unblock someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:bookmarks" +msgstr "Add to or remove from your bookmarks" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:chats" +msgstr "Create or delete chats or chat messages, or mark them as read" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:conversations" +msgstr "Change recipients of, mark as read, or delete conversations" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:favourites" +msgstr "Favourite or unfavourite statuses" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:filters" +msgstr "Change your filtering settings" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follow" +msgstr "Follow or unfollow someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follows" +msgstr "Follow or unfollow someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:lists" +msgstr "Create, change or delete your lists" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:media" +msgstr "Upload media files or modify those you uploaded" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:mutes" +msgstr "Mute or unmute someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:notifications" +msgstr "Mark notifications as read" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:reports" +msgstr "Submit reports" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:statuses" +msgstr "Post, edit, reblog or react to statuses" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:accounts" +msgstr "Read all accounts using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:chats" +msgstr "Read all chats using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:invites" +msgstr "Read all invites using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:media_proxy_caches" +msgstr "Read media proxy caches using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:reports" +msgstr "Read all reports using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:statuses" +msgstr "Read all statuses using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:accounts" +msgstr "Change all accounts using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:chats" +msgstr "Change all chats using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:follows" +msgstr "Change follow relationships using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:invites" +msgstr "Invite or revoke an invite using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:media_proxy_caches" +msgstr "Change media proxy caches using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:reports" +msgstr "Handle reports using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:statuses" +msgstr "Delete, change scope of, or mark as sensitive statuses using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:media" +msgstr "Read media attachments" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:mutes" +msgstr "Read mute relationships" From 99f157e2804327d4ad3f47b83638060f774a582b Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 22:12:15 -0400 Subject: [PATCH 20/35] Fix MR pipelines not having build and test jobs --- .gitlab-ci.yml | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2eee3bc688..b70415ac62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,24 +56,27 @@ check-changelog: - apk add git - sh ./tools/check-changelog +.build_changes_policy: + rules: + - changes: + - ".gitlab-ci.yml" + - "**/*.ex" + - "**/*.exs" + - "mix.lock" + build: + extends: .build_changes_policy stage: build - only: - changes: &build_changes_policy - - ".gitlab-ci.yml" - - "**/*.ex" - - "**/*.exs" - - "mix.lock" script: - mix compile --force spec-build: stage: test - only: - changes: - - ".gitlab-ci.yml" - - "lib/pleroma/web/api_spec/**/*.ex" - - "lib/pleroma/web/api_spec.ex" + rules: + - changes: + - ".gitlab-ci.yml" + - "lib/pleroma/web/api_spec/**/*.ex" + - "lib/pleroma/web/api_spec.ex" artifacts: paths: - spec.json @@ -95,9 +98,8 @@ benchmark: - mix pleroma.load_testing unit-testing: + extends: .build_changes_policy stage: test - only: - changes: *build_changes_policy cache: &testing_cache_policy <<: *global_cache_policy policy: pull @@ -118,11 +120,10 @@ unit-testing: path: coverage.xml unit-testing-erratic: + extends: .build_changes_policy stage: test retry: 2 allow_failure: true - only: - changes: *build_changes_policy cache: &testing_cache_policy <<: *global_cache_policy policy: pull @@ -153,9 +154,8 @@ unit-testing-erratic: # - mix test --trace --only federated unit-testing-rum: + extends: .build_changes_policy stage: test - only: - changes: *build_changes_policy cache: *testing_cache_policy services: - name: minibikini/postgres-with-rum:12 @@ -171,10 +171,9 @@ unit-testing-rum: - mix test --preload-modules lint: + extends: .build_changes_policy image: ¤t_elixir elixir:1.12-alpine stage: test - only: - changes: *build_changes_policy cache: *testing_cache_policy before_script: ¤t_bfr_script - apk update @@ -186,18 +185,16 @@ lint: - mix format --check-formatted analysis: + extends: .build_changes_policy stage: test - only: - changes: *build_changes_policy cache: *testing_cache_policy script: - mix credo --strict --only=warnings,todo,fixme,consistency,readability cycles: + extends: .build_changes_policy image: *current_elixir stage: test - only: - changes: *build_changes_policy cache: {} before_script: *current_bfr_script script: From 6a3fd8e01404f7fc4415626300d81c65c045a7d6 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 22:16:00 -0400 Subject: [PATCH 21/35] Do not count for renames when diffing --- tools/check-changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check-changelog b/tools/check-changelog index 1409be9f01..60692033f4 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -5,7 +5,7 @@ echo "looking for change log" git remote add upstream https://git.pleroma.social/pleroma/pleroma.git git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}:refs/remotes/upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -git diff --raw upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ +git diff --raw --no-renames upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\)$' ret=$? From 163e82bab1cc12ad323bfc9c1fcada11fa3b223a Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 9 May 2023 19:25:39 -0400 Subject: [PATCH 22/35] Allow lang attribute --- changelog.d/3882.add | 1 + priv/scrubbers/default.ex | 77 ++++++++++++++-------------- test/pleroma/web/common_api_test.exs | 11 ++++ 3 files changed, 51 insertions(+), 38 deletions(-) create mode 100644 changelog.d/3882.add diff --git a/changelog.d/3882.add b/changelog.d/3882.add new file mode 100644 index 0000000000..4712de1dc7 --- /dev/null +++ b/changelog.d/3882.add @@ -0,0 +1 @@ +Allow lang attribute in status text diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index e10e3ec87d..d1215d2e0b 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -33,35 +33,35 @@ defmodule Pleroma.HTML.Scrubber.Default do "ugc" ]) - Meta.allow_tag_with_these_attributes(:a, ["name", "title"]) + Meta.allow_tag_with_these_attributes(:a, ["name", "title", "lang"]) - Meta.allow_tag_with_these_attributes(:abbr, ["title"]) + Meta.allow_tag_with_these_attributes(:abbr, ["title", "lang"]) - Meta.allow_tag_with_these_attributes(:b, []) - Meta.allow_tag_with_these_attributes(:blockquote, []) - Meta.allow_tag_with_these_attributes(:br, []) - Meta.allow_tag_with_these_attributes(:code, []) - Meta.allow_tag_with_these_attributes(:del, []) - Meta.allow_tag_with_these_attributes(:em, []) - Meta.allow_tag_with_these_attributes(:hr, []) - Meta.allow_tag_with_these_attributes(:i, []) - Meta.allow_tag_with_these_attributes(:li, []) - Meta.allow_tag_with_these_attributes(:ol, []) - Meta.allow_tag_with_these_attributes(:p, []) - Meta.allow_tag_with_these_attributes(:pre, []) - Meta.allow_tag_with_these_attributes(:strong, []) - Meta.allow_tag_with_these_attributes(:sub, []) - Meta.allow_tag_with_these_attributes(:sup, []) - Meta.allow_tag_with_these_attributes(:ruby, []) - Meta.allow_tag_with_these_attributes(:rb, []) - Meta.allow_tag_with_these_attributes(:rp, []) - Meta.allow_tag_with_these_attributes(:rt, []) - Meta.allow_tag_with_these_attributes(:rtc, []) - Meta.allow_tag_with_these_attributes(:u, []) - Meta.allow_tag_with_these_attributes(:ul, []) + Meta.allow_tag_with_these_attributes(:b, ["lang"]) + Meta.allow_tag_with_these_attributes(:blockquote, ["lang"]) + Meta.allow_tag_with_these_attributes(:br, ["lang"]) + Meta.allow_tag_with_these_attributes(:code, ["lang"]) + Meta.allow_tag_with_these_attributes(:del, ["lang"]) + Meta.allow_tag_with_these_attributes(:em, ["lang"]) + Meta.allow_tag_with_these_attributes(:hr, ["lang"]) + Meta.allow_tag_with_these_attributes(:i, ["lang"]) + Meta.allow_tag_with_these_attributes(:li, ["lang"]) + Meta.allow_tag_with_these_attributes(:ol, ["lang"]) + Meta.allow_tag_with_these_attributes(:p, ["lang"]) + Meta.allow_tag_with_these_attributes(:pre, ["lang"]) + Meta.allow_tag_with_these_attributes(:strong, ["lang"]) + Meta.allow_tag_with_these_attributes(:sub, ["lang"]) + Meta.allow_tag_with_these_attributes(:sup, ["lang"]) + Meta.allow_tag_with_these_attributes(:ruby, ["lang"]) + Meta.allow_tag_with_these_attributes(:rb, ["lang"]) + Meta.allow_tag_with_these_attributes(:rp, ["lang"]) + Meta.allow_tag_with_these_attributes(:rt, ["lang"]) + Meta.allow_tag_with_these_attributes(:rtc, ["lang"]) + Meta.allow_tag_with_these_attributes(:u, ["lang"]) + Meta.allow_tag_with_these_attributes(:ul, ["lang"]) Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "recipients-inline"]) - Meta.allow_tag_with_these_attributes(:span, []) + Meta.allow_tag_with_these_attributes(:span, ["lang"]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) @@ -77,29 +77,30 @@ defmodule Pleroma.HTML.Scrubber.Default do "width", "height", "title", - "alt" + "alt", + "lang" ]) end if Pleroma.Config.get([:markup, :allow_tables]) do - Meta.allow_tag_with_these_attributes(:table, []) - Meta.allow_tag_with_these_attributes(:tbody, []) - Meta.allow_tag_with_these_attributes(:td, []) - Meta.allow_tag_with_these_attributes(:th, []) - Meta.allow_tag_with_these_attributes(:thead, []) - Meta.allow_tag_with_these_attributes(:tr, []) + Meta.allow_tag_with_these_attributes(:table, ["lang"]) + Meta.allow_tag_with_these_attributes(:tbody, ["lang"]) + Meta.allow_tag_with_these_attributes(:td, ["lang"]) + Meta.allow_tag_with_these_attributes(:th, ["lang"]) + Meta.allow_tag_with_these_attributes(:thead, ["lang"]) + Meta.allow_tag_with_these_attributes(:tr, ["lang"]) end if Pleroma.Config.get([:markup, :allow_headings]) do - Meta.allow_tag_with_these_attributes(:h1, []) - Meta.allow_tag_with_these_attributes(:h2, []) - Meta.allow_tag_with_these_attributes(:h3, []) - Meta.allow_tag_with_these_attributes(:h4, []) - Meta.allow_tag_with_these_attributes(:h5, []) + Meta.allow_tag_with_these_attributes(:h1, ["lang"]) + Meta.allow_tag_with_these_attributes(:h2, ["lang"]) + Meta.allow_tag_with_these_attributes(:h3, ["lang"]) + Meta.allow_tag_with_these_attributes(:h4, ["lang"]) + Meta.allow_tag_with_these_attributes(:h5, ["lang"]) end if Pleroma.Config.get([:markup, :allow_fonts]) do - Meta.allow_tag_with_these_attributes(:font, ["face"]) + Meta.allow_tag_with_these_attributes(:font, ["face", "lang"]) end Meta.strip_everything_not_covered() diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 5c9103e9fc..968d826a2d 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -527,6 +527,17 @@ test "zwnj is treated as word character" do assert Object.tags(object) == ["ساٴين‌س"] end + test "allows lang attribute" do + user = insert(:user) + text = ~s{something

random

} + + {:ok, activity} = CommonAPI.post(user, %{status: text, content_type: "text/html"}) + + object = Object.normalize(activity, fetch: false) + + assert object.data["content"] == text + end + test "double dot in link is allowed" do user = insert(:user) text = "https://example.to/something..mp3" From be5c5118cb296792d82c007fe3b83d8795dca4b0 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 9 May 2023 21:04:01 -0400 Subject: [PATCH 23/35] Make sure object refetching follows update rules --- changelog.d/3883.fix | 1 + lib/pleroma/object/fetcher.ex | 86 ++++---------------- lib/pleroma/object/updater.ex | 49 +++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 36 ++------ test/pleroma/object/fetcher_test.exs | 84 ++++++++++++++++--- 5 files changed, 145 insertions(+), 111 deletions(-) create mode 100644 changelog.d/3883.fix diff --git a/changelog.d/3883.fix b/changelog.d/3883.fix new file mode 100644 index 0000000000..6824f20134 --- /dev/null +++ b/changelog.d/3883.fix @@ -0,0 +1 @@ +Fix abnormal behaviour when refetching a poll diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index a9a9eeeed1..cc3772563b 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -8,77 +8,30 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment - alias Pleroma.Repo alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Federator require Logger require Pleroma.Constants - defp touch_changeset(changeset) do - updated_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.truncate(:second) - - Ecto.Changeset.put_change(changeset, :updated_at, updated_at) - end - - defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do - has_history? = fn - %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true - _ -> false - end - - internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) - - remote_history_exists? = has_history?.(new_data) - - # If the remote history exists, we treat that as the only source of truth. - new_data = - if has_history?.(old_data) and not remote_history_exists? do - Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"]) - else - new_data - end - - # If the remote does not have history information, we need to manage it ourselves - new_data = - if not remote_history_exists? do - changed? = - Pleroma.Constants.status_updatable_fields() - |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) - - %{updated_object: updated_object} = - new_data - |> Object.Updater.maybe_update_history(old_data, - updated: changed?, - use_history_in_new_object?: false - ) - - updated_object - else - new_data - end - - Map.merge(new_data, internal_fields) - end - - defp maybe_reinject_internal_fields(_, new_data), do: new_data - @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} - defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do + defp reinject_object(%Object{data: %{}} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") - with data <- maybe_reinject_internal_fields(object, new_data), - {:ok, data, _} <- ObjectValidator.validate(data, %{}), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do - {:ok, object} + with {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}), + {:ok, new_data} <- MRF.filter(new_data), + {:ok, new_object, _} <- + Object.Updater.do_update_and_invalidate_cache( + object, + new_data, + _touch_changeset? = true + ) do + {:ok, new_object} else e -> Logger.error("Error while processing object: #{inspect(e)}") @@ -86,20 +39,11 @@ defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) end end - defp reinject_object(%Object{} = object, new_data) do - Logger.debug("Reinjecting object #{new_data["id"]}") - - with new_data <- Transmogrifier.fix_object(new_data), - data <- maybe_reinject_internal_fields(object, new_data), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do + defp reinject_object(_, new_data) do + with {:ok, object, _} <- Pipeline.common_pipeline(new_data, local: false) do {:ok, object} else - e -> - Logger.error("Error while processing object: #{inspect(e)}") - {:error, e} + e -> e end end diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index ab38d3ed2b..0c523ed350 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -5,6 +5,9 @@ defmodule Pleroma.Object.Updater do require Pleroma.Constants + alias Pleroma.Object + alias Pleroma.Repo + def update_content_fields(orig_object_data, updated_object) do Pleroma.Constants.status_updatable_fields() |> Enum.reduce( @@ -237,4 +240,50 @@ def do_with_history(object, fun) do {:history_items, e} -> e end end + + defp maybe_touch_changeset(changeset, true) do + updated_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + + Ecto.Changeset.put_change(changeset, :updated_at, updated_at) + end + + defp maybe_touch_changeset(changeset, _), do: changeset + + def do_update_and_invalidate_cache(orig_object, updated_object, touch_changeset? \\ false) do + orig_object_ap_id = updated_object["id"] + orig_object = Object.get_by_ap_id(orig_object_ap_id) + orig_object_data = orig_object.data + + %{ + updated_data: updated_object_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = make_new_object_data_from_update_object(orig_object_data, updated_object) + + changeset = + orig_object + |> Repo.preload(:hashtags) + |> Object.change(%{data: updated_object_data}) + |> maybe_touch_changeset(touch_changeset?) + + with {:ok, new_object} <- Repo.update(changeset), + {:ok, _} <- Object.invalid_object_cache(new_object), + {:ok, _} <- Object.set_cache(new_object), + # The metadata/utils.ex uses the object id for the cache. + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if used_history_in_new_object? do + with create_activity when not is_nil(create_activity) <- + Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do + nil + else + _ -> nil + end + end + + {:ok, new_object, updated} + end + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index e19642d50a..098c177c7a 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -428,37 +428,13 @@ defp handle_update_object( end if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do - %{ - updated_data: updated_object_data, - updated: updated, - used_history_in_new_object?: used_history_in_new_object? - } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object) + {:ok, _, updated} = + Object.Updater.do_update_and_invalidate_cache(orig_object, updated_object) - changeset = - orig_object - |> Repo.preload(:hashtags) - |> Object.change(%{data: updated_object_data}) - - with {:ok, new_object} <- Repo.update(changeset), - {:ok, _} <- Object.invalid_object_cache(new_object), - {:ok, _} <- Object.set_cache(new_object), - # The metadata/utils.ex uses the object id for the cache. - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do - if used_history_in_new_object? do - with create_activity when not is_nil(create_activity) <- - Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do - nil - else - _ -> nil - end - end - - if updated do - object - |> Activity.normalize() - |> ActivityPub.notify_and_stream() - end + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() end end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index c8ad66ddb4..53c9277d67 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -9,8 +9,12 @@ defmodule Pleroma.Object.FetcherTest do alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Object.Fetcher + alias Pleroma.Web.ActivityPub.ObjectValidator + + require Pleroma.Constants import Mock + import Pleroma.Factory import Tesla.Mock setup do @@ -284,6 +288,8 @@ test "it can refetch pruned objects" do describe "refetching" do setup do + insert(:user, ap_id: "https://mastodon.social/users/emelie") + object1 = %{ "id" => "https://mastodon.social/1", "actor" => "https://mastodon.social/users/emelie", @@ -293,10 +299,14 @@ test "it can refetch pruned objects" do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:20Z" } + {:ok, local_object1, _} = ObjectValidator.validate(object1, []) + object2 = %{ "id" => "https://mastodon.social/2", "actor" => "https://mastodon.social/users/emelie", @@ -306,8 +316,10 @@ test "it can refetch pruned objects" do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], + "to" => [Pleroma.Constants.as_public()], "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:25Z", "formerRepresentations" => %{ "type" => "OrderedCollection", "orderedItems" => [ @@ -319,14 +331,18 @@ test "it can refetch pruned objects" do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:21Z" } ], "totalItems" => 1 } } + {:ok, local_object2, _} = ObjectValidator.validate(object2, []) + mock(fn %{ method: :get, @@ -335,7 +351,7 @@ test "it can refetch pruned objects" do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object1) + body: Jason.encode!(object1 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -345,7 +361,7 @@ test "it can refetch pruned objects" do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object2) + body: Jason.encode!(object2 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -370,7 +386,7 @@ test "it can refetch pruned objects" do apply(HttpRequestMock, :request, [env]) end) - %{object1: object1, object2: object2} + %{object1: local_object1, object2: local_object2} end test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do @@ -388,8 +404,9 @@ test "it keeps formerRepresentations if remote does not have this attr", %{objec "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z" } ], "totalItems" => 1 @@ -467,6 +484,53 @@ test "it adds to formerRepresentations if the remote does not have one and the o } } = refetched.data end + + test "it keeps the history intact if only updated time has changed", + %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "updated" => "2023-05-08 23:43:47Z", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 1"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 1", + "formerRepresentations" => %{ + "orderedItems" => [ + %{"content" => "mew mew 1"} + ], + "totalItems" => 1 + } + } = refetched.data + end + + test "it goes through ObjectValidator and MRF", %{object2: object2} do + with_mock Pleroma.Web.ActivityPub.MRF, [:passthrough], + filter: fn + %{"type" => "Note"} = object -> + {:ok, Map.put(object, "content", "MRFd content")} + + arg -> + passthrough([arg]) + end do + {:ok, o} = Object.create(object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"content" => "MRFd content"} = refetched.data + end + end end describe "fetch with history" do From e170fc40dd46b2cfa4970c5d8174bdcfd6cf821c Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 9 May 2023 21:38:28 -0400 Subject: [PATCH 24/35] Fix build warning --- lib/pleroma/object/updater.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 0c523ed350..bad8119651 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -253,7 +253,6 @@ defp maybe_touch_changeset(changeset, _), do: changeset def do_update_and_invalidate_cache(orig_object, updated_object, touch_changeset? \\ false) do orig_object_ap_id = updated_object["id"] - orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data %{ From 505e58d4eb84226613c85ac526c591dab5b1307e Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 22 May 2023 08:14:20 -0400 Subject: [PATCH 25/35] Fix ObjectTest --- lib/pleroma/object/updater.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index bad8119651..269609f1bd 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -100,12 +100,14 @@ def maybe_update_history( end defp maybe_update_poll(to_be_updated, updated_object) do - choice_key = fn data -> - if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + choice_key = fn + %{"anyOf" => [_ | _]} = data -> "anyOf" + %{"oneOf" => [_ | _]} = data -> "oneOf" + _ -> nil end with true <- to_be_updated["type"] == "Question", - key <- choice_key.(updated_object), + key when not is_nil(key) <- choice_key.(updated_object), true <- key == choice_key.(to_be_updated), orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), From 6aafa7fe7661bcf9e62c16381ffeb2d32ec9bafa Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 22 May 2023 08:16:14 -0400 Subject: [PATCH 26/35] Add changelog --- changelog.d/fix-object-test.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fix-object-test.fix diff --git a/changelog.d/fix-object-test.fix b/changelog.d/fix-object-test.fix new file mode 100644 index 0000000000..5eea719f0b --- /dev/null +++ b/changelog.d/fix-object-test.fix @@ -0,0 +1 @@ +Correctly handle the situation when a poll has both "anyOf" and "oneOf" but one of them being empty From 819a82da99588b53a67f42d8572a2484c4f17d40 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 22 May 2023 08:19:58 -0400 Subject: [PATCH 27/35] Fix unused variable --- lib/pleroma/object/updater.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 269609f1bd..b1e4870bac 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -101,8 +101,8 @@ def maybe_update_history( defp maybe_update_poll(to_be_updated, updated_object) do choice_key = fn - %{"anyOf" => [_ | _]} = data -> "anyOf" - %{"oneOf" => [_ | _]} = data -> "oneOf" + %{"anyOf" => [_ | _]} -> "anyOf" + %{"oneOf" => [_ | _]} -> "oneOf" _ -> nil end From 38bcf6b19e3d83cb6c4e6c82d237a26edcab167a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 May 2023 12:08:50 -0400 Subject: [PATCH 28/35] MediaProxyController: Apply CSP sandbox --- changelog.d/3126.fix | 1 + .../web/media_proxy/media_proxy_controller.ex | 7 +++++++ .../media_proxy/media_proxy_controller_test.exs | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 changelog.d/3126.fix diff --git a/changelog.d/3126.fix b/changelog.d/3126.fix new file mode 100644 index 0000000000..91d396c89d --- /dev/null +++ b/changelog.d/3126.fix @@ -0,0 +1 @@ +MediaProxy responses now return a sandbox CSP header diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index d2ad62c139..bda5b36edc 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do alias Pleroma.Web.MediaProxy alias Plug.Conn + plug(:sandbox) + def remote(conn, %{"sig" => sig64, "url" => url64}) do with {_, true} <- {:enabled, MediaProxy.enabled?()}, {:ok, url} <- MediaProxy.decode_url(sig64, url64), @@ -202,4 +204,9 @@ defp media_preview_proxy_config do defp media_proxy_opts do Config.get([:media_proxy, :proxy_opts], []) end + + defp sandbox(conn, _params) do + conn + |> merge_resp_headers([{"content-security-policy", "sandbox;"}]) + end end diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 5246bf0c4b..9ce092fd8f 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -6,7 +6,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do use Pleroma.Web.ConnCase import Mock + import Mox + alias Pleroma.ReverseProxy.ClientMock alias Pleroma.Web.MediaProxy alias Plug.Conn @@ -74,6 +76,20 @@ test "it returns 404 when url is in banned_urls cache", %{conn: conn, url: url} assert %Conn{status: 404, resp_body: "Not Found"} = get(conn, url) end end + + test "it applies sandbox CSP to MediaProxy requests", %{conn: conn} do + media_url = "https://lain.com/image.png" + media_proxy_url = MediaProxy.encode_url(media_url) + + ClientMock + |> expect(:request, fn :get, ^media_url, _, _, _ -> + {:ok, 200, [{"content-type", "image/png"}]} + end) + + %Conn{resp_headers: headers} = get(conn, media_proxy_url) + + assert {"content-security-policy", "sandbox;"} in headers + end end describe "Media Preview Proxy" do From 72833c84b583057ed0e757f35b65f90af40496df Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Wed, 17 May 2023 18:50:35 +0000 Subject: [PATCH 29/35] Merge branch 'tusooa/rework-refetch' into 'develop' Make sure object refetching follows update rules See merge request pleroma/pleroma!3883 --- changelog.d/3883.fix | 1 + lib/pleroma/object/fetcher.ex | 86 ++++---------------- lib/pleroma/object/updater.ex | 48 +++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 36 ++------ test/pleroma/object/fetcher_test.exs | 84 ++++++++++++++++--- 5 files changed, 144 insertions(+), 111 deletions(-) create mode 100644 changelog.d/3883.fix diff --git a/changelog.d/3883.fix b/changelog.d/3883.fix new file mode 100644 index 0000000000..6824f20134 --- /dev/null +++ b/changelog.d/3883.fix @@ -0,0 +1 @@ +Fix abnormal behaviour when refetching a poll diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index a9a9eeeed1..cc3772563b 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -8,77 +8,30 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment - alias Pleroma.Repo alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Federator require Logger require Pleroma.Constants - defp touch_changeset(changeset) do - updated_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.truncate(:second) - - Ecto.Changeset.put_change(changeset, :updated_at, updated_at) - end - - defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do - has_history? = fn - %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true - _ -> false - end - - internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) - - remote_history_exists? = has_history?.(new_data) - - # If the remote history exists, we treat that as the only source of truth. - new_data = - if has_history?.(old_data) and not remote_history_exists? do - Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"]) - else - new_data - end - - # If the remote does not have history information, we need to manage it ourselves - new_data = - if not remote_history_exists? do - changed? = - Pleroma.Constants.status_updatable_fields() - |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) - - %{updated_object: updated_object} = - new_data - |> Object.Updater.maybe_update_history(old_data, - updated: changed?, - use_history_in_new_object?: false - ) - - updated_object - else - new_data - end - - Map.merge(new_data, internal_fields) - end - - defp maybe_reinject_internal_fields(_, new_data), do: new_data - @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} - defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do + defp reinject_object(%Object{data: %{}} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") - with data <- maybe_reinject_internal_fields(object, new_data), - {:ok, data, _} <- ObjectValidator.validate(data, %{}), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do - {:ok, object} + with {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}), + {:ok, new_data} <- MRF.filter(new_data), + {:ok, new_object, _} <- + Object.Updater.do_update_and_invalidate_cache( + object, + new_data, + _touch_changeset? = true + ) do + {:ok, new_object} else e -> Logger.error("Error while processing object: #{inspect(e)}") @@ -86,20 +39,11 @@ defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) end end - defp reinject_object(%Object{} = object, new_data) do - Logger.debug("Reinjecting object #{new_data["id"]}") - - with new_data <- Transmogrifier.fix_object(new_data), - data <- maybe_reinject_internal_fields(object, new_data), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do + defp reinject_object(_, new_data) do + with {:ok, object, _} <- Pipeline.common_pipeline(new_data, local: false) do {:ok, object} else - e -> - Logger.error("Error while processing object: #{inspect(e)}") - {:error, e} + e -> e end end diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index ab38d3ed2b..bad8119651 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -5,6 +5,9 @@ defmodule Pleroma.Object.Updater do require Pleroma.Constants + alias Pleroma.Object + alias Pleroma.Repo + def update_content_fields(orig_object_data, updated_object) do Pleroma.Constants.status_updatable_fields() |> Enum.reduce( @@ -237,4 +240,49 @@ def do_with_history(object, fun) do {:history_items, e} -> e end end + + defp maybe_touch_changeset(changeset, true) do + updated_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + + Ecto.Changeset.put_change(changeset, :updated_at, updated_at) + end + + defp maybe_touch_changeset(changeset, _), do: changeset + + def do_update_and_invalidate_cache(orig_object, updated_object, touch_changeset? \\ false) do + orig_object_ap_id = updated_object["id"] + orig_object_data = orig_object.data + + %{ + updated_data: updated_object_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = make_new_object_data_from_update_object(orig_object_data, updated_object) + + changeset = + orig_object + |> Repo.preload(:hashtags) + |> Object.change(%{data: updated_object_data}) + |> maybe_touch_changeset(touch_changeset?) + + with {:ok, new_object} <- Repo.update(changeset), + {:ok, _} <- Object.invalid_object_cache(new_object), + {:ok, _} <- Object.set_cache(new_object), + # The metadata/utils.ex uses the object id for the cache. + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if used_history_in_new_object? do + with create_activity when not is_nil(create_activity) <- + Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do + nil + else + _ -> nil + end + end + + {:ok, new_object, updated} + end + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index a2152b9452..fc5dec3628 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -428,37 +428,13 @@ defp handle_update_object( end if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do - %{ - updated_data: updated_object_data, - updated: updated, - used_history_in_new_object?: used_history_in_new_object? - } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object) + {:ok, _, updated} = + Object.Updater.do_update_and_invalidate_cache(orig_object, updated_object) - changeset = - orig_object - |> Repo.preload(:hashtags) - |> Object.change(%{data: updated_object_data}) - - with {:ok, new_object} <- Repo.update(changeset), - {:ok, _} <- Object.invalid_object_cache(new_object), - {:ok, _} <- Object.set_cache(new_object), - # The metadata/utils.ex uses the object id for the cache. - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do - if used_history_in_new_object? do - with create_activity when not is_nil(create_activity) <- - Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do - nil - else - _ -> nil - end - end - - if updated do - object - |> Activity.normalize() - |> ActivityPub.notify_and_stream() - end + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() end end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index c8ad66ddb4..53c9277d67 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -9,8 +9,12 @@ defmodule Pleroma.Object.FetcherTest do alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Object.Fetcher + alias Pleroma.Web.ActivityPub.ObjectValidator + + require Pleroma.Constants import Mock + import Pleroma.Factory import Tesla.Mock setup do @@ -284,6 +288,8 @@ test "it can refetch pruned objects" do describe "refetching" do setup do + insert(:user, ap_id: "https://mastodon.social/users/emelie") + object1 = %{ "id" => "https://mastodon.social/1", "actor" => "https://mastodon.social/users/emelie", @@ -293,10 +299,14 @@ test "it can refetch pruned objects" do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:20Z" } + {:ok, local_object1, _} = ObjectValidator.validate(object1, []) + object2 = %{ "id" => "https://mastodon.social/2", "actor" => "https://mastodon.social/users/emelie", @@ -306,8 +316,10 @@ test "it can refetch pruned objects" do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], + "to" => [Pleroma.Constants.as_public()], "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:25Z", "formerRepresentations" => %{ "type" => "OrderedCollection", "orderedItems" => [ @@ -319,14 +331,18 @@ test "it can refetch pruned objects" do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:21Z" } ], "totalItems" => 1 } } + {:ok, local_object2, _} = ObjectValidator.validate(object2, []) + mock(fn %{ method: :get, @@ -335,7 +351,7 @@ test "it can refetch pruned objects" do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object1) + body: Jason.encode!(object1 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -345,7 +361,7 @@ test "it can refetch pruned objects" do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object2) + body: Jason.encode!(object2 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -370,7 +386,7 @@ test "it can refetch pruned objects" do apply(HttpRequestMock, :request, [env]) end) - %{object1: object1, object2: object2} + %{object1: local_object1, object2: local_object2} end test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do @@ -388,8 +404,9 @@ test "it keeps formerRepresentations if remote does not have this attr", %{objec "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z" } ], "totalItems" => 1 @@ -467,6 +484,53 @@ test "it adds to formerRepresentations if the remote does not have one and the o } } = refetched.data end + + test "it keeps the history intact if only updated time has changed", + %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "updated" => "2023-05-08 23:43:47Z", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 1"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 1", + "formerRepresentations" => %{ + "orderedItems" => [ + %{"content" => "mew mew 1"} + ], + "totalItems" => 1 + } + } = refetched.data + end + + test "it goes through ObjectValidator and MRF", %{object2: object2} do + with_mock Pleroma.Web.ActivityPub.MRF, [:passthrough], + filter: fn + %{"type" => "Note"} = object -> + {:ok, Map.put(object, "content", "MRFd content")} + + arg -> + passthrough([arg]) + end do + {:ok, o} = Object.create(object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"content" => "MRFd content"} = refetched.data + end + end end describe "fetch with history" do From 4339230f64b05fee1c4d7313c1dc9adc45827a5d Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 23 May 2023 01:57:07 +0000 Subject: [PATCH 30/35] Merge branch 'tusooa/fix-object-test' into 'develop' Fix ObjectTest See merge request pleroma/pleroma!3887 --- changelog.d/fix-object-test.fix | 1 + lib/pleroma/object/updater.ex | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fix-object-test.fix diff --git a/changelog.d/fix-object-test.fix b/changelog.d/fix-object-test.fix new file mode 100644 index 0000000000..5eea719f0b --- /dev/null +++ b/changelog.d/fix-object-test.fix @@ -0,0 +1 @@ +Correctly handle the situation when a poll has both "anyOf" and "oneOf" but one of them being empty diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index bad8119651..b1e4870bac 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -100,12 +100,14 @@ def maybe_update_history( end defp maybe_update_poll(to_be_updated, updated_object) do - choice_key = fn data -> - if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + choice_key = fn + %{"anyOf" => [_ | _]} -> "anyOf" + %{"oneOf" => [_ | _]} -> "oneOf" + _ -> nil end with true <- to_be_updated["type"] == "Question", - key <- choice_key.(updated_object), + key when not is_nil(key) <- choice_key.(updated_object), true <- key == choice_key.(to_be_updated), orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), From b36263e5ffd0d89d819b01478f19891b14740bb0 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Fri, 26 May 2023 17:12:18 +0000 Subject: [PATCH 31/35] Merge branch 'issue/3126' into 'develop' MediaProxyController: Apply CSP sandbox See merge request pleroma/pleroma!3890 --- changelog.d/3126.fix | 1 + .../web/media_proxy/media_proxy_controller.ex | 7 +++++++ .../media_proxy/media_proxy_controller_test.exs | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 changelog.d/3126.fix diff --git a/changelog.d/3126.fix b/changelog.d/3126.fix new file mode 100644 index 0000000000..91d396c89d --- /dev/null +++ b/changelog.d/3126.fix @@ -0,0 +1 @@ +MediaProxy responses now return a sandbox CSP header diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index d2ad62c139..bda5b36edc 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do alias Pleroma.Web.MediaProxy alias Plug.Conn + plug(:sandbox) + def remote(conn, %{"sig" => sig64, "url" => url64}) do with {_, true} <- {:enabled, MediaProxy.enabled?()}, {:ok, url} <- MediaProxy.decode_url(sig64, url64), @@ -202,4 +204,9 @@ defp media_preview_proxy_config do defp media_proxy_opts do Config.get([:media_proxy, :proxy_opts], []) end + + defp sandbox(conn, _params) do + conn + |> merge_resp_headers([{"content-security-policy", "sandbox;"}]) + end end diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 5246bf0c4b..9ce092fd8f 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -6,7 +6,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do use Pleroma.Web.ConnCase import Mock + import Mox + alias Pleroma.ReverseProxy.ClientMock alias Pleroma.Web.MediaProxy alias Plug.Conn @@ -74,6 +76,20 @@ test "it returns 404 when url is in banned_urls cache", %{conn: conn, url: url} assert %Conn{status: 404, resp_body: "Not Found"} = get(conn, url) end end + + test "it applies sandbox CSP to MediaProxy requests", %{conn: conn} do + media_url = "https://lain.com/image.png" + media_proxy_url = MediaProxy.encode_url(media_url) + + ClientMock + |> expect(:request, fn :get, ^media_url, _, _, _ -> + {:ok, 200, [{"content-type", "image/png"}]} + end) + + %Conn{resp_headers: headers} = get(conn, media_proxy_url) + + assert {"content-security-policy", "sandbox;"} in headers + end end describe "Media Preview Proxy" do From d0c2e0830b16c541d2883004f071a0954be45682 Mon Sep 17 00:00:00 2001 From: tusooa Date: Thu, 30 Mar 2023 21:01:37 -0400 Subject: [PATCH 32/35] Enforce unauth restrictions for public streaming endpoints --- lib/pleroma/web/streamer.ex | 46 +++++++++++++++--- test/pleroma/web/streamer_test.exs | 77 ++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 3c0da5c276..b9a04cc767 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Web.Streamer do def registry, do: @registry @public_streams ["public", "public:local", "public:media", "public:local:media"] + @local_streams ["public:local", "public:local:media"] @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"] @doc "Expands and authorizes a stream, and registers the process for streaming." @@ -41,14 +42,37 @@ def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do end end + defp can_access_stream(user, oauth_token, kind) do + with {_, true} <- {:restrict?, Config.restrict_unauthenticated_access?(:timelines, kind)}, + {_, %User{id: user_id}, %Token{user_id: user_id}} <- {:user, user, oauth_token}, + {_, true} <- + {:scopes, + OAuthScopesPlug.filter_descendants(["read:statuses"], oauth_token.scopes) != []} do + true + else + {:restrict?, _} -> + true + + _ -> + false + end + end + @doc "Expand and authorizes a stream" @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) :: {:ok, topic :: String.t()} | {:error, :bad_topic} def get_topic(stream, user, oauth_token, params \\ %{}) - # Allow all public steams. - def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do - {:ok, stream} + # Allow all public steams if the instance allows unauthenticated access. + # Otherwise, only allow users with valid oauth tokens. + def get_topic(stream, user, oauth_token, _params) when stream in @public_streams do + kind = if stream in @local_streams, do: :local, else: :federated + + if can_access_stream(user, oauth_token, kind) do + {:ok, stream} + else + {:error, :unauthorized} + end end # Allow all hashtags streams. @@ -57,12 +81,20 @@ def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do end # Allow remote instance streams. - def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do - {:ok, "public:remote:" <> instance} + def get_topic("public:remote", user, oauth_token, %{"instance" => instance} = _params) do + if can_access_stream(user, oauth_token, :federated) do + {:ok, "public:remote:" <> instance} + else + {:error, :unauthorized} + end end - def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do - {:ok, "public:remote:media:" <> instance} + def get_topic("public:remote:media", user, oauth_token, %{"instance" => instance} = _params) do + if can_access_stream(user, oauth_token, :federated) do + {:ok, "public:remote:media:" <> instance} + else + {:error, :unauthorized} + end end # Expand user streams. diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 8b0c84164d..7ab0e379b4 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -29,6 +29,26 @@ test "allows public" do assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil) end + test "rejects local public streams if restricted_unauthenticated is on" do + clear_config([:restrict_unauthenticated, :timelines, :local], true) + + assert {:error, :unauthorized} = Streamer.get_topic("public:local", nil, nil) + assert {:error, :unauthorized} = Streamer.get_topic("public:local:media", nil, nil) + end + + test "rejects remote public streams if restricted_unauthenticated is on" do + clear_config([:restrict_unauthenticated, :timelines, :federated], true) + + assert {:error, :unauthorized} = Streamer.get_topic("public", nil, nil) + assert {:error, :unauthorized} = Streamer.get_topic("public:media", nil, nil) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"}) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote:media", nil, nil, %{"instance" => "lain.com"}) + end + test "allows instance streams" do assert {:ok, "public:remote:lain.com"} = Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"}) @@ -69,6 +89,63 @@ test "allows public streams (regardless of OAuth token scopes)", %{ end end + test "allows local public streams if restricted_unauthenticated is on", %{ + user: user, + token: oauth_token + } do + clear_config([:restrict_unauthenticated, :timelines, :local], true) + + %{token: read_notifications_token} = oauth_access(["read:notifications"], user: user) + %{token: badly_scoped_token} = oauth_access(["irrelevant:scope"], user: user) + + assert {:ok, "public:local"} = Streamer.get_topic("public:local", user, oauth_token) + + assert {:ok, "public:local:media"} = + Streamer.get_topic("public:local:media", user, oauth_token) + + for token <- [read_notifications_token, badly_scoped_token] do + assert {:error, :unauthorized} = Streamer.get_topic("public:local", user, token) + + assert {:error, :unauthorized} = Streamer.get_topic("public:local:media", user, token) + end + end + + test "allows remote public streams if restricted_unauthenticated is on", %{ + user: user, + token: oauth_token + } do + clear_config([:restrict_unauthenticated, :timelines, :federated], true) + + %{token: read_notifications_token} = oauth_access(["read:notifications"], user: user) + %{token: badly_scoped_token} = oauth_access(["irrelevant:scope"], user: user) + + assert {:ok, "public"} = Streamer.get_topic("public", user, oauth_token) + assert {:ok, "public:media"} = Streamer.get_topic("public:media", user, oauth_token) + + assert {:ok, "public:remote:lain.com"} = + Streamer.get_topic("public:remote", user, oauth_token, %{"instance" => "lain.com"}) + + assert {:ok, "public:remote:media:lain.com"} = + Streamer.get_topic("public:remote:media", user, oauth_token, %{ + "instance" => "lain.com" + }) + + for token <- [read_notifications_token, badly_scoped_token] do + assert {:error, :unauthorized} = Streamer.get_topic("public", user, token) + assert {:error, :unauthorized} = Streamer.get_topic("public:media", user, token) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote", user, token, %{ + "instance" => "lain.com" + }) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote:media", user, token, %{ + "instance" => "lain.com" + }) + end + end + test "allows user streams (with proper OAuth token scopes)", %{ user: user, token: read_oauth_token From 0d68804aa7efc4f3212e02218804755da93d03f0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 May 2023 13:23:59 -0400 Subject: [PATCH 33/35] Filter OEmbed HTML tags --- changelog.d/3891.fix | 1 + lib/pleroma/web/rich_media/parsers/o_embed.ex | 4 ++-- test/pleroma/web/rich_media/parser_test.exs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/3891.fix diff --git a/changelog.d/3891.fix b/changelog.d/3891.fix new file mode 100644 index 0000000000..f1fb62d826 --- /dev/null +++ b/changelog.d/3891.fix @@ -0,0 +1 @@ +OEmbed HTML tags are now filtered diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex index 75318d9c72..0f303176ce 100644 --- a/lib/pleroma/web/rich_media/parsers/o_embed.ex +++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do def parse(html, _data) do with elements = [_ | _] <- get_discovery_data(html), oembed_url when is_binary(oembed_url) <- get_oembed_url(elements), - {:ok, oembed_data} <- get_oembed_data(oembed_url) do - oembed_data + {:ok, oembed_data = %{"html" => html}} <- get_oembed_data(oembed_url) do + %{oembed_data | "html" => Pleroma.HTML.filter_tags(html)} else _e -> %{} end diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs index ffdc4e5d78..9064138a64 100644 --- a/test/pleroma/web/rich_media/parser_test.exs +++ b/test/pleroma/web/rich_media/parser_test.exs @@ -129,7 +129,7 @@ test "parses twitter card" do }} end - test "parses OEmbed" do + test "parses OEmbed and filters HTML tags" do assert Parser.parse("http://example.com/oembed") == {:ok, %{ @@ -139,7 +139,7 @@ test "parses OEmbed" do "flickr_type" => "photo", "height" => "768", "html" => - "\"Bacon", + "\"Bacon", "license" => "All Rights Reserved", "license_id" => 0, "provider_name" => "Flickr", From 4505bc1e587ce76683e21a41f90fb2956ed2a084 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 May 2023 13:23:59 -0400 Subject: [PATCH 34/35] Filter OEmbed HTML tags --- changelog.d/3891.fix | 1 + lib/pleroma/web/rich_media/parsers/o_embed.ex | 4 ++-- test/pleroma/web/rich_media/parser_test.exs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/3891.fix diff --git a/changelog.d/3891.fix b/changelog.d/3891.fix new file mode 100644 index 0000000000..f1fb62d826 --- /dev/null +++ b/changelog.d/3891.fix @@ -0,0 +1 @@ +OEmbed HTML tags are now filtered diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex index 75318d9c72..0f303176ce 100644 --- a/lib/pleroma/web/rich_media/parsers/o_embed.ex +++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do def parse(html, _data) do with elements = [_ | _] <- get_discovery_data(html), oembed_url when is_binary(oembed_url) <- get_oembed_url(elements), - {:ok, oembed_data} <- get_oembed_data(oembed_url) do - oembed_data + {:ok, oembed_data = %{"html" => html}} <- get_oembed_data(oembed_url) do + %{oembed_data | "html" => Pleroma.HTML.filter_tags(html)} else _e -> %{} end diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs index ffdc4e5d78..9064138a64 100644 --- a/test/pleroma/web/rich_media/parser_test.exs +++ b/test/pleroma/web/rich_media/parser_test.exs @@ -129,7 +129,7 @@ test "parses twitter card" do }} end - test "parses OEmbed" do + test "parses OEmbed and filters HTML tags" do assert Parser.parse("http://example.com/oembed") == {:ok, %{ @@ -139,7 +139,7 @@ test "parses OEmbed" do "flickr_type" => "photo", "height" => "768", "html" => - "\"Bacon", + "\"Bacon", "license" => "All Rights Reserved", "license_id" => 0, "provider_name" => "Flickr", From 7618e562b34d4240514d8819407dd6bd3adbe79d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 30 Mar 2023 12:52:57 +0200 Subject: [PATCH 35/35] Version 2.5.2 --- CHANGELOG.md | 19 +++++++++++++++++++ mix.exs | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1742ee2f4d..f6fc6aaee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed +### Removed + +## 2.5.2 + +### Security +- `/proxy` endpoint now sets a Content-Security-Policy (sandbox) +- WebSocket endpoint now respects unauthenticated restrictions for streams of public posts +- OEmbed HTML tags are now filtered + +### Changed +- docs: Be more explicit about the level of compatibility of OTP releases +- Set default background worker timeout to 15 minutes + +### Fixed +- Atom/RSS formatting (HTML truncation, published, missing summary) +- Remove `static_fe` pipeline for `/users/:nickname/feed` +- Stop oban from retrying if validating errors occur when processing incoming data +- Make sure object refetching as used by already received polls follows MRF rules + ### Removed - BREAKING: Support for passwords generated with `crypt(3)` (Gnu Social migration artifact) diff --git a/mix.exs b/mix.exs index 7439bbca65..79fd9c9efe 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("2.5.1"), + version: version("2.5.2"), elixir: "~> 1.11", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(),