From b0b674f47e49e3631341958a4c70fc255f7bbf42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 7 Sep 2024 19:07:16 +0200 Subject: [PATCH 001/100] Support migration from Akkoma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- docs/installation/migrating_from_akkoma.md | 15 ++++++ .../20230306112859_instances_add_metadata.exs | 4 +- .../20220108213213_add_mastofe_settings.exs | 14 ++++++ .../20220718102634_upgrade_oban_to_v11.exs | 5 ++ ...emove_remote_cancelled_follow_requests.exs | 5 ++ ...1170605_remove_local_cancelled_follows.exs | 5 ++ ...20911195347_add_user_frontend_profiles.exs | 17 +++++++ ...20220916115149_ensure_mastofe_settings.exs | 14 ++++++ .../20221020135943_add_nodeinfo.exs | 15 ++++++ ...21123221956_add_has_request_signatures.exs | 12 +++++ ...0221128103145_add_per_user_post_expiry.exs | 12 +++++ ...331_add_notification_activity_id_index.exs | 5 ++ ...110627_add_bookmarks_activity_id_index.exs | 5 ++ ...727_add_report_notes_activity_id_index.exs | 5 ++ ...ade_to_report_notes_on_activity_delete.exs | 16 ++++++ ...0221203232118_add_user_follows_hashtag.exs | 17 +++++++ .../20221211234352_remove_unused_indices.exs | 5 ++ ..._ap_id_coalesce_follower_address_index.exs | 15 ++++++ ...2213837_add_unfollowed_dm_restrictions.exs | 12 +++++ .../20240210000000_drop_chat_tables.exs | 49 +++++++++++++++++++ .../20240213120000_add_permit_followback.exs | 12 +++++ ...iftool_to_exiftool_strip_location_real.exs | 34 +++++++++++++ 22 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 docs/installation/migrating_from_akkoma.md create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20220718102634_upgrade_oban_to_v11.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20220805123645_remove_remote_cancelled_follow_requests.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20220831170605_remove_local_cancelled_follows.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20220911195347_add_user_frontend_profiles.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20220916115149_ensure_mastofe_settings.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221020135943_add_nodeinfo.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221123221956_add_has_request_signatures.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221128103145_add_per_user_post_expiry.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221129105331_add_notification_activity_id_index.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221129110627_add_bookmarks_activity_id_index.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221129110727_add_report_notes_activity_id_index.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221211234352_remove_unused_indices.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20230127143303_rename_index_users_ap_id_coalesce_follower_address_index.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20230522213837_add_unfollowed_dm_restrictions.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20240210000000_drop_chat_tables.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs create mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20240425120000_upload_filter_exiftool_to_exiftool_strip_location_real.exs diff --git a/docs/installation/migrating_from_akkoma.md b/docs/installation/migrating_from_akkoma.md new file mode 100644 index 0000000000..901608b82c --- /dev/null +++ b/docs/installation/migrating_from_akkoma.md @@ -0,0 +1,15 @@ +# Migrating from Akkoma + +## Database migration + +To rollback Akkoma-specific migrations: + +- OTP: `./bin/pleroma_ctl rollback --migrations-path priv/repo/optional_migrations/akkoma_rollbacks` +- From Source: `mix ecto.rollback --migrations-path priv/repo/optional_migrations/akkoma_rollbacks` + +Then, just + +- OTP: `./bin/pleroma_ctl migrate` +- From Source: `mix ecto.migrate` + +to apply Pleroma database migrations. \ No newline at end of file diff --git a/priv/repo/migrations/20230306112859_instances_add_metadata.exs b/priv/repo/migrations/20230306112859_instances_add_metadata.exs index 898f5220e0..6af9fd3300 100644 --- a/priv/repo/migrations/20230306112859_instances_add_metadata.exs +++ b/priv/repo/migrations/20230306112859_instances_add_metadata.exs @@ -7,8 +7,8 @@ defmodule Pleroma.Repo.Migrations.InstancesAddMetadata do def change do alter table(:instances) do - add(:metadata, :map) - add(:metadata_updated_at, :utc_datetime) + add_if_not_exists(:metadata, :map) + add_if_not_exists(:metadata_updated_at, :utc_datetime) end end end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs new file mode 100644 index 0000000000..7f98807fb6 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs @@ -0,0 +1,14 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20220108213213_add_mastofe_settings.exs + +defmodule Pleroma.Repo.Migrations.AddMastofeSettings do + use Ecto.Migration + + def up, do: :ok + + def down do + alter table(:users) do + remove_if_exists(:mastofe_settings, :map) + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20220718102634_upgrade_oban_to_v11.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20220718102634_upgrade_oban_to_v11.exs new file mode 100644 index 0000000000..eb85978afc --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20220718102634_upgrade_oban_to_v11.exs @@ -0,0 +1,5 @@ +defmodule Pleroma.Repo.Migrations.UpgradeObanToV11 do + use Ecto.Migration + + def change, do: :ok +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20220805123645_remove_remote_cancelled_follow_requests.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20220805123645_remove_remote_cancelled_follow_requests.exs new file mode 100644 index 0000000000..b94a5ab2ed --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20220805123645_remove_remote_cancelled_follow_requests.exs @@ -0,0 +1,5 @@ +defmodule Pleroma.Repo.Migrations.RemoveRemoteCancelledFollowRequests do + use Ecto.Migration + + def change, do: :ok +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20220831170605_remove_local_cancelled_follows.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20220831170605_remove_local_cancelled_follows.exs new file mode 100644 index 0000000000..0289c2a063 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20220831170605_remove_local_cancelled_follows.exs @@ -0,0 +1,5 @@ +defmodule Pleroma.Repo.Migrations.RemoveLocalCancelledFollows do + use Ecto.Migration + + def change, do: :ok +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20220911195347_add_user_frontend_profiles.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20220911195347_add_user_frontend_profiles.exs new file mode 100644 index 0000000000..11c50d1a3c --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20220911195347_add_user_frontend_profiles.exs @@ -0,0 +1,17 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20220911195347_add_user_frontend_profiles.exs + +defmodule Pleroma.Repo.Migrations.AddUserFrontendProfiles do + use Ecto.Migration + + def up, do: :ok + + def down do + drop_if_exists(table("user_frontend_setting_profiles")) + drop_if_exists(index(:user_frontend_setting_profiles, [:user_id, :frontend_name])) + + drop_if_exists( + unique_index(:user_frontend_setting_profiles, [:user_id, :frontend_name, :profile_name]) + ) + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20220916115149_ensure_mastofe_settings.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20220916115149_ensure_mastofe_settings.exs new file mode 100644 index 0000000000..3074051cc8 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20220916115149_ensure_mastofe_settings.exs @@ -0,0 +1,14 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20220916115149_ensure_mastofe_settings.exs + +defmodule Pleroma.Repo.Migrations.EnsureMastofeSettings do + use Ecto.Migration + + def up, do: :ok + + def down do + alter table(:users) do + remove_if_exists(:mastofe_settings, :map) + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221020135943_add_nodeinfo.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221020135943_add_nodeinfo.exs new file mode 100644 index 0000000000..8afdee76ed --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221020135943_add_nodeinfo.exs @@ -0,0 +1,15 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20221020135943_add_nodeinfo.exs + +defmodule Pleroma.Repo.Migrations.AddNodeinfo do + use Ecto.Migration + + def up, do: :ok + + def down do + alter table(:instances) do + remove_if_exists(:nodeinfo, :map) + remove_if_exists(:metadata_updated_at, :naive_datetime) + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221123221956_add_has_request_signatures.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221123221956_add_has_request_signatures.exs new file mode 100644 index 0000000000..ce51a6bba2 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221123221956_add_has_request_signatures.exs @@ -0,0 +1,12 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20221123221956_add_has_request_signatures.exs + +defmodule Pleroma.Repo.Migrations.AddHasRequestSignatures do + use Ecto.Migration + + def change do + alter table(:instances) do + add(:has_request_signatures, :boolean, default: false, null: false) + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221128103145_add_per_user_post_expiry.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221128103145_add_per_user_post_expiry.exs new file mode 100644 index 0000000000..f7a5f28358 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221128103145_add_per_user_post_expiry.exs @@ -0,0 +1,12 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20221128103145_add_per_user_post_expiry.exs + +defmodule Pleroma.Repo.Migrations.AddPerUserPostExpiry do + use Ecto.Migration + + def change do + alter table(:users) do + add(:status_ttl_days, :integer, null: true) + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221129105331_add_notification_activity_id_index.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221129105331_add_notification_activity_id_index.exs new file mode 100644 index 0000000000..c23cef5f63 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221129105331_add_notification_activity_id_index.exs @@ -0,0 +1,5 @@ +defmodule Pleroma.Repo.Migrations.AddNotificationActivityIdIndex do + use Ecto.Migration + + def change, do: :ok +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221129110627_add_bookmarks_activity_id_index.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221129110627_add_bookmarks_activity_id_index.exs new file mode 100644 index 0000000000..3a5a1277ec --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221129110627_add_bookmarks_activity_id_index.exs @@ -0,0 +1,5 @@ +defmodule Pleroma.Repo.Migrations.AddBookmarksActivityIdIndex do + use Ecto.Migration + + def change, do: :ok +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221129110727_add_report_notes_activity_id_index.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221129110727_add_report_notes_activity_id_index.exs new file mode 100644 index 0000000000..92e0df7e0a --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221129110727_add_report_notes_activity_id_index.exs @@ -0,0 +1,5 @@ +defmodule Pleroma.Repo.Migrations.AddReportNotesActivityIdIndex do + use Ecto.Migration + + def change, do: :ok +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs new file mode 100644 index 0000000000..41d871bbaa --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs @@ -0,0 +1,16 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs + +defmodule Pleroma.Repo.Migrations.AddCascadeToReportNotesOnActivityDelete do + use Ecto.Migration + + def up, do: :ok + + def down do + drop(constraint(:report_notes, "report_notes_activity_id_fkey")) + + alter table(:report_notes) do + modify(:activity_id, references(:activities, type: :uuid)) + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs new file mode 100644 index 0000000000..ed8c1a0bc5 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs @@ -0,0 +1,17 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20221203232118_add_user_follows_hashtag.exs + +defmodule Pleroma.Repo.Migrations.AddUserFollowsHashtag do + use Ecto.Migration + + def change do + create table(:user_follows_hashtag) do + add(:hashtag_id, references(:hashtags)) + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + end + + create(unique_index(:user_follows_hashtag, [:user_id, :hashtag_id])) + + create_if_not_exists(index(:user_follows_hashtag, [:hashtag_id])) + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221211234352_remove_unused_indices.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221211234352_remove_unused_indices.exs new file mode 100644 index 0000000000..77797aaab1 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20221211234352_remove_unused_indices.exs @@ -0,0 +1,5 @@ +defmodule Pleroma.Repo.Migrations.RemoveUnusedIndices do + use Ecto.Migration + + def change, do: :ok +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20230127143303_rename_index_users_ap_id_coalesce_follower_address_index.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20230127143303_rename_index_users_ap_id_coalesce_follower_address_index.exs new file mode 100644 index 0000000000..54c389e274 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20230127143303_rename_index_users_ap_id_coalesce_follower_address_index.exs @@ -0,0 +1,15 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20230127143303_rename_index_users_ap_id_coalesce_follower_address_index.exs + +defmodule Pleroma.Repo.Migrations.RenameIndexUsersApId_COALESCEFollowerAddressIndex do + alias Pleroma.Repo + + use Ecto.Migration + + def up, do: :ok + + def down do + Repo.query!("ALTER INDEX public.\"aa_users_ap_id_COALESCE_follower_address_index\" + RENAME TO \"users_ap_id_COALESCE_follower_address_index\";") + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20230522213837_add_unfollowed_dm_restrictions.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20230522213837_add_unfollowed_dm_restrictions.exs new file mode 100644 index 0000000000..39d260dd2a --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20230522213837_add_unfollowed_dm_restrictions.exs @@ -0,0 +1,12 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20230522213837_add_unfollowed_dm_restrictions.exs + +defmodule Pleroma.Repo.Migrations.AddUnfollowedDmRestrictions do + use Ecto.Migration + + def change do + alter table(:users) do + add(:accepts_direct_messages_from, :string, default: "everybody") + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20240210000000_drop_chat_tables.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20240210000000_drop_chat_tables.exs new file mode 100644 index 0000000000..8d1d0aa2f5 --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20240210000000_drop_chat_tables.exs @@ -0,0 +1,49 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20240210000000_drop_chat_tables.exs + +defmodule Pleroma.Repo.Migrations.DropChatTables do + use Ecto.Migration + + def up, do: :ok + + def down do + # Ecto's default primary key is bigserial, thus configure manually + create table(:chats, primary_key: false) do + add(:id, :uuid, primary_key: true, autogenerated: true) + + add( + :user_id, + references(:users, type: :uuid, on_delete: :delete_all) + # yes, this was nullable + ) + + add( + :recipient, + references(:users, column: :ap_id, type: :string, on_delete: :delete_all) + # yes, this was nullable + ) + + timestamps() + end + + create(index(:chats, [:user_id, :recipient], unique: true)) + + create table(:chat_message_references, primary_key: false) do + add(:id, :uuid, primary_key: true, autogenerated: true) + add(:chat_id, references(:chats, type: :uuid, on_delete: :delete_all), null: false) + add(:object_id, references(:objects, on_delete: :delete_all), null: false) + add(:unread, :boolean, default: true, null: false) + timestamps() + end + + create(index(:chat_message_references, [:chat_id, "id desc"])) + create(unique_index(:chat_message_references, [:object_id, :chat_id])) + + create( + index(:chat_message_references, [:chat_id], + where: "unread = true", + name: "unread_messages_count_index" + ) + ) + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs new file mode 100644 index 0000000000..9b3834a4ab --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs @@ -0,0 +1,12 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20240213120000_add_permit_followback.exs + +defmodule Pleroma.Repo.Migrations.AddPermitFollowback do + use Ecto.Migration + + def change do + alter table(:users) do + add(:permit_followback, :boolean, null: false, default: false) + end + end +end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20240425120000_upload_filter_exiftool_to_exiftool_strip_location_real.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20240425120000_upload_filter_exiftool_to_exiftool_strip_location_real.exs new file mode 100644 index 0000000000..48c675476d --- /dev/null +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20240425120000_upload_filter_exiftool_to_exiftool_strip_location_real.exs @@ -0,0 +1,34 @@ +# Adapted from Akkoma +# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20240425120000_upload_filter_exiftool_to_exiftool_strip_location_real.exs + +defmodule Pleroma.Repo.Migrations.UploadFilterExiftoolToExiftoolStripMetadataReal do + use Ecto.Migration + + alias Pleroma.ConfigDB + + def up, do: :ok + + def down, + do: + ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload}) + |> update_filtername( + Pleroma.Upload.Filter.Exiftool.StripMetadata, + Pleroma.Upload.Filter.Exiftool + ) + + defp update_filtername(%{value: value}, from_filtername, to_filtername) do + new_value = + value + |> Keyword.update(:filters, [], fn filters -> + filters + |> Enum.map(fn + ^from_filtername -> to_filtername + filter -> filter + end) + end) + + ConfigDB.update_or_create(%{group: :pleroma, key: Pleroma.Upload, value: new_value}) + end + + defp update_filtername(_, _, _), do: nil +end From ecf2c06e35d839b89f46d61099df34d1932e4f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 7 Sep 2024 19:12:09 +0200 Subject: [PATCH 002/100] Akkoma rollbacks: Remove migrations used by pl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../20221203232118_add_user_follows_hashtag.exs | 17 ----------------- .../20240213120000_add_permit_followback.exs | 12 ------------ 2 files changed, 29 deletions(-) delete mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs delete mode 100644 priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs deleted file mode 100644 index ed8c1a0bc5..0000000000 --- a/priv/repo/optional_migrations/akkoma_rollbacks/20221203232118_add_user_follows_hashtag.exs +++ /dev/null @@ -1,17 +0,0 @@ -# Adapted from Akkoma -# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20221203232118_add_user_follows_hashtag.exs - -defmodule Pleroma.Repo.Migrations.AddUserFollowsHashtag do - use Ecto.Migration - - def change do - create table(:user_follows_hashtag) do - add(:hashtag_id, references(:hashtags)) - add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) - end - - create(unique_index(:user_follows_hashtag, [:user_id, :hashtag_id])) - - create_if_not_exists(index(:user_follows_hashtag, [:hashtag_id])) - end -end diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs deleted file mode 100644 index 9b3834a4ab..0000000000 --- a/priv/repo/optional_migrations/akkoma_rollbacks/20240213120000_add_permit_followback.exs +++ /dev/null @@ -1,12 +0,0 @@ -# Adapted from Akkoma -# https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/priv/repo/migrations/20240213120000_add_permit_followback.exs - -defmodule Pleroma.Repo.Migrations.AddPermitFollowback do - use Ecto.Migration - - def change do - alter table(:users) do - add(:permit_followback, :boolean, null: false, default: false) - end - end -end From 687e67b711db2b45241558db02866fd9c8115f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 7 Sep 2024 19:33:10 +0200 Subject: [PATCH 003/100] Update docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/akkoma-migration.add | 1 + docs/installation/migrating_from_akkoma.md | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/akkoma-migration.add diff --git a/changelog.d/akkoma-migration.add b/changelog.d/akkoma-migration.add new file mode 100644 index 0000000000..c8b457ec23 --- /dev/null +++ b/changelog.d/akkoma-migration.add @@ -0,0 +1 @@ +Add instructions for migrating from Akkoma \ No newline at end of file diff --git a/docs/installation/migrating_from_akkoma.md b/docs/installation/migrating_from_akkoma.md index 901608b82c..2a486e1f6c 100644 --- a/docs/installation/migrating_from_akkoma.md +++ b/docs/installation/migrating_from_akkoma.md @@ -2,6 +2,8 @@ ## Database migration +> Note: You will lose data related about Akkoma-specific features, including: MastoFE settings, user frontend profiles, status auto-expiration config, hashtag follows, DM restrictions and auto follow-back. Consider taking a backup. + To rollback Akkoma-specific migrations: - OTP: `./bin/pleroma_ctl rollback --migrations-path priv/repo/optional_migrations/akkoma_rollbacks` From 8213d3579205fb92e3206a03896af2f5aedc3b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 7 Sep 2024 19:33:52 +0200 Subject: [PATCH 004/100] update akkoma migration docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- docs/installation/migrating_from_akkoma.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/migrating_from_akkoma.md b/docs/installation/migrating_from_akkoma.md index 2a486e1f6c..563eb9763d 100644 --- a/docs/installation/migrating_from_akkoma.md +++ b/docs/installation/migrating_from_akkoma.md @@ -2,7 +2,7 @@ ## Database migration -> Note: You will lose data related about Akkoma-specific features, including: MastoFE settings, user frontend profiles, status auto-expiration config, hashtag follows, DM restrictions and auto follow-back. Consider taking a backup. +> Note: You will lose data related about Akkoma-specific features, including: MastoFE settings, user frontend profiles, status auto-expiration config and DM restrictions. Consider taking a backup. To rollback Akkoma-specific migrations: From c9b28eaf9a484fa1f2c27d00855c997575369782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 8 Sep 2024 05:23:46 +0300 Subject: [PATCH 005/100] Argon2 password support --- lib/pleroma/web/plugs/authentication_plug.ex | 4 ++++ mix.exs | 1 + mix.lock | 1 + 3 files changed, 6 insertions(+) diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex index f912a1542f..3fc6e7b516 100644 --- a/lib/pleroma/web/plugs/authentication_plug.ex +++ b/lib/pleroma/web/plugs/authentication_plug.ex @@ -47,6 +47,10 @@ def checkpw(password, "$pbkdf2" <> _ = password_hash) do Pleroma.Password.Pbkdf2.verify_pass(password, password_hash) end + def checkpw(password, "$argon2" <> _ = password_hash) do + Argon2.verify_pass(password, password_hash) + end + def checkpw(_password, _password_hash) do Logger.error("Password hash not recognized") false diff --git a/mix.exs b/mix.exs index df44934d7b..0d49a6b45f 100644 --- a/mix.exs +++ b/mix.exs @@ -203,6 +203,7 @@ defp deps do {:websock_adapter, "~> 0.5.6"}, {:oban_live_dashboard, "~> 0.1.1"}, {:multipart, "~> 0.4.0", optional: true}, + {:argon2_elixir, "~> 4.0"}, ## dev & test {:phoenix_live_reload, "~> 1.3.3", only: :dev}, diff --git a/mix.lock b/mix.lock index 865e09a4c7..01f2eef98e 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,6 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, + "argon2_elixir": {:hex, :argon2_elixir, "4.0.0", "7f6cd2e4a93a37f61d58a367d82f830ad9527082ff3c820b8197a8a736648941", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f9da27cf060c9ea61b1bd47837a28d7e48a8f6fa13a745e252556c14f9132c7f"}, "bandit": {:hex, :bandit, "1.5.5", "df28f1c41f745401fe9e85a6882033f5f3442ab6d30c8a2948554062a4ab56e0", [:mix], [{:hpax, "~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "f21579a29ea4bc08440343b2b5f16f7cddf2fea5725d31b72cf973ec729079e1"}, "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"}, "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"}, From 9de522ce5048bd72dd083a1661506b563be27cc1 Mon Sep 17 00:00:00 2001 From: Mint Date: Sun, 8 Sep 2024 05:32:40 +0300 Subject: [PATCH 006/100] Authentication: convert argon2 passwords, add tests --- lib/pleroma/web/plugs/authentication_plug.ex | 5 ++++ .../web/plugs/authentication_plug_test.exs | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex index 3fc6e7b516..af7d7f45a8 100644 --- a/lib/pleroma/web/plugs/authentication_plug.ex +++ b/lib/pleroma/web/plugs/authentication_plug.ex @@ -48,6 +48,7 @@ def checkpw(password, "$pbkdf2" <> _ = password_hash) do end def checkpw(password, "$argon2" <> _ = password_hash) do + # Handle argon2 passwords for Akkoma migration Argon2.verify_pass(password, password_hash) end @@ -60,6 +61,10 @@ def maybe_update_password(%User{password_hash: "$2" <> _} = user, password) do do_update_password(user, password) end + def maybe_update_password(%User{password_hash: "$argon2" <> _} = user, password) do + do_update_password(user, password) + end + def maybe_update_password(user, _), do: {:ok, user} defp do_update_password(user, password) do diff --git a/test/pleroma/web/plugs/authentication_plug_test.exs b/test/pleroma/web/plugs/authentication_plug_test.exs index b8acd01c59..bdbf3de320 100644 --- a/test/pleroma/web/plugs/authentication_plug_test.exs +++ b/test/pleroma/web/plugs/authentication_plug_test.exs @@ -70,6 +70,24 @@ test "with a bcrypt hash, it updates to a pkbdf2 hash", %{conn: conn} do assert "$pbkdf2" <> _ = user.password_hash end + test "with an argon2 hash, it updates to a pkbdf2 hash", %{conn: conn} do + user = insert(:user, password_hash: Argon2.hash_pwd_salt("123")) + assert "$argon2" <> _ = user.password_hash + + conn = + conn + |> assign(:auth_user, user) + |> assign(:auth_credentials, %{password: "123"}) + |> AuthenticationPlug.call(%{}) + + assert conn.assigns.user.id == conn.assigns.auth_user.id + assert conn.assigns.token == nil + assert PlugHelper.plug_skipped?(conn, OAuthScopesPlug) + + user = User.get_by_id(user.id) + assert "$pbkdf2" <> _ = user.password_hash + end + describe "checkpw/2" do test "check pbkdf2 hash" do hash = @@ -86,6 +104,14 @@ test "check bcrypt hash" do refute AuthenticationPlug.checkpw("password1", hash) end + test "check argon2 hash" do + hash = + "$argon2id$v=19$m=65536,t=8,p=2$zEMMsTuK5KkL5AFWbX7jyQ$VyaQD7PF6e9btz0oH1YiAkWwIGZ7WNDZP8l+a/O171g" + + assert AuthenticationPlug.checkpw("password", hash) + refute AuthenticationPlug.checkpw("password1", hash) + end + test "it returns false when hash invalid" do hash = "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" From 7e91c3a306b8f050e6a88e888fd439b579c1f125 Mon Sep 17 00:00:00 2001 From: Mint Date: Sun, 8 Sep 2024 05:41:48 +0300 Subject: [PATCH 007/100] Changelog --- changelog.d/argon2-passwords.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/argon2-passwords.add diff --git a/changelog.d/argon2-passwords.add b/changelog.d/argon2-passwords.add new file mode 100644 index 0000000000..36fd7faf22 --- /dev/null +++ b/changelog.d/argon2-passwords.add @@ -0,0 +1 @@ +Added support for argon2 passwords and their conversion for migration from Akkoma fork to upstream. From b09152801a2c4628d07ccd61e279cc3cfeac2758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 8 Sep 2024 13:22:16 +0200 Subject: [PATCH 008/100] Add localBubbleInstances to nodeinfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/nodeinfo/nodeinfo.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo.ex b/lib/pleroma/web/nodeinfo/nodeinfo.ex index 612bfd3bcc..c86d8823c5 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo.ex @@ -75,6 +75,7 @@ def get_nodeinfo("2.0") do restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), skipThreadContainment: Config.get([:instance, :skip_thread_containment], false), federatedTimelineAvailable: Config.get([:instance, :federated_timeline_available], true), + localBubbleInstances: Config.get([:instance, :local_bubble], []), publicTimelineVisibility: %{ federated: !Config.restrict_unauthenticated_access?(:timelines, :federated) && From efbd25d0b57e69fa095737b5c526d3b0f9fe664d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 8 Sep 2024 15:40:42 +0200 Subject: [PATCH 009/100] Support Akkoma translation routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/akkoma_compat_controller.ex | 100 ++++++++++++++++++ .../operations/akkoma_compat_operation.ex | 91 ++++++++++++++++ .../web/mastodon_api/views/instance_view.ex | 6 +- lib/pleroma/web/router.ex | 7 ++ .../web/akkoma_compat_controller_test.exs | 60 +++++++++++ 5 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/web/akkoma_compat_controller.ex create mode 100644 lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex create mode 100644 test/pleroma/web/akkoma_compat_controller_test.exs diff --git a/lib/pleroma/web/akkoma_compat_controller.ex b/lib/pleroma/web/akkoma_compat_controller.ex new file mode 100644 index 0000000000..57abc29d05 --- /dev/null +++ b/lib/pleroma/web/akkoma_compat_controller.ex @@ -0,0 +1,100 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AkkomaCompatController do + use Pleroma.Web, :controller + + alias Pleroma.Activity + alias Pleroma.Language.Translation + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(:skip_auth when action == :translation_languages) + + plug( + OAuthScopesPlug, + %{fallback: :proceed_unauthenticated, scopes: ["read:statuses"]} when action == :translate + ) + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.AkkomaCompatOperation + + @doc "GET /api/v1/akkoma/translation/languages" + def translation_languages(conn, _params) do + with {:enabled, true} <- {:enabled, Translation.configured?()}, + {:ok, source_languages} <- Translation.supported_languages(:source), + {:ok, target_languages} <- Translation.supported_languages(:target) do + source_languages = + source_languages + |> Enum.map(fn lang -> %{code: lang, name: lang} end) + + target_languages = + target_languages + |> Enum.map(fn lang -> %{code: lang, name: lang} end) + + conn + |> json(%{source: source_languages, target: target_languages}) + else + {:enabled, false} -> + json(conn, %{}) + + e -> + {:error, e} + end + end + + @doc "GET /api/v1/statuses/:id/translations/:language" + def translate( + %{ + assigns: %{user: user}, + private: %{open_api_spex: %{params: %{id: status_id} = params}} + } = conn, + _ + ) do + with {:authentication, true} <- + {:authentication, + !is_nil(user) || + Pleroma.Config.get([Translation, :allow_unauthenticated])}, + %Activity{object: object} <- Activity.get_by_id_with_object(status_id), + {:visibility, visibility} when visibility in ["public", "unlisted"] <- + {:visibility, Visibility.get_visibility(object)}, + {:allow_remote, true} <- + {:allow_remote, + Object.local?(object) || + Pleroma.Config.get([Translation, :allow_remote])}, + {:language, language} when is_binary(language) <- + {:language, Map.get(params, :language) || user.language}, + {:ok, result} <- + Translation.translate( + object.data["content"], + object.data["language"], + language + ) do + json(conn, %{detected_language: result.detected_source_language, text: result.content}) + else + {:authentication, false} -> + render_error(conn, :unauthorized, "Authorization is required to translate statuses") + + {:allow_remote, false} -> + render_error(conn, :bad_request, "You can't translate remote posts") + + {:language, nil} -> + render_error(conn, :bad_request, "Language not specified") + + {:visibility, _} -> + render_error(conn, :not_found, "Record not found") + + {:error, :not_found} -> + render_error(conn, :not_found, "Translation service not configured") + + {:error, error} when error in [:unexpected_response, :quota_exceeded, :too_many_requests] -> + render_error(conn, :service_unavailable, "Translation service not available") + + nil -> + render_error(conn, :not_found, "Record not found") + end + end +end diff --git a/lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex b/lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex new file mode 100644 index 0000000000..83d29caf9b --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex @@ -0,0 +1,91 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.AkkomaCompatOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + # Adapted from https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/lib/pleroma/web/api_spec/operations/translate_operation.ex + def translation_languages_operation() do + %Operation{ + tags: ["Akkoma compatibility routes"], + summary: "Get translation languages", + description: "Retreieve a list of supported source and target language", + operationId: "AkkomaCompatController.translation_languages", + responses: %{ + 200 => + Operation.response( + "Translation languages", + "application/json", + source_dest_languages_schema() + ) + } + } + end + + defp source_dest_languages_schema do + %Schema{ + type: :object, + required: [:source, :target], + properties: %{ + source: languages_schema(), + target: languages_schema() + } + } + end + + defp languages_schema do + %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + code: %Schema{type: :string}, + name: %Schema{type: :string} + } + } + } + end + + def translate_operation() do + %Operation{ + tags: ["Akkoma compatibility routes"], + summary: "Translate status", + description: "Translate status with an external API", + operationId: "AkkomaCompatController.translate", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [ + Operation.parameter(:id, :path, FlakeID.schema(), "Status ID", + example: "9umDrYheeY451cQnEe", + required: true + ), + Operation.parameter(:language, :path, :string, "Target language code", example: "en"), + Operation.parameter(:from, :query, :string, "Source language code (unused)", + example: "en" + ) + ], + responses: %{ + 200 => + Operation.response( + "Translated status", + "application/json", + %Schema{ + type: :object, + required: [:detected_language, :text], + properties: %{ + detected_language: %Schema{type: :string}, + text: %Schema{type: :string} + } + } + ) + } + } + end +end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 81ee1a0a6f..8d0171bf3d 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -182,7 +182,11 @@ def features do end, "events", "multitenancy", - "pleroma:bites" + "pleroma:bites", + # Akkoma compatibility + if Pleroma.Language.Translation.configured?() do + "akkoma:machine_translation" + end ] |> Enum.filter(& &1) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9bb3e07ba9..8fcabcd20d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -718,6 +718,13 @@ defmodule Pleroma.Web.Router do end end + scope "/", Pleroma.Web do + pipe_through(:api) + + get("/api/v1/akkoma/translation/languages", AkkomaCompatController, :translation_languages) + get("/api/v1/statuses/:id/translations/:language", AkkomaCompatController, :translate) + end + scope "/api/v1", Pleroma.Web.MastodonAPI do pipe_through(:authenticated_api) diff --git a/test/pleroma/web/akkoma_compat_controller_test.exs b/test/pleroma/web/akkoma_compat_controller_test.exs new file mode 100644 index 0000000000..02b691ca0f --- /dev/null +++ b/test/pleroma/web/akkoma_compat_controller_test.exs @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AkkomaCompatControllerTest do + use Pleroma.Web.ConnCase, async: true + + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory + + describe "translation_languages" do + test "returns supported languages list", %{conn: conn} do + clear_config([Pleroma.Language.Translation, :provider], TranslationMock) + + assert %{ + "source" => [%{"code" => "en", "name" => "en"}, %{"code" => "pl", "name" => "pl"}], + "target" => [%{"code" => "en", "name" => "en"}, %{"code" => "pl", "name" => "pl"}] + } = + conn + |> get("/api/v1/akkoma/translation/languages") + |> json_response_and_validate_schema(200) + end + + test "returns empty object when disabled", %{conn: conn} do + clear_config([Pleroma.Language.Translation, :provider], nil) + + assert %{} == + conn + |> get("/api/v1/akkoma/translation/languages") + |> json_response(200) + end + end + + describe "translate" do + test "it translates a status to given language" do + clear_config([Pleroma.Language.Translation, :provider], TranslationMock) + + %{conn: conn} = oauth_access(["read:statuses"]) + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "Cześć!", + visibility: "public", + language: "pl" + }) + + response = + conn + |> get("/api/v1/statuses/#{activity.id}/translations/en") + |> json_response_and_validate_schema(200) + + assert response == %{ + "text" => "!ćśezC", + "detected_language" => "pl" + } + end + end +end From 8e12a2ca2597f06c84c1912d2b1f39eb065f43e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 8 Sep 2024 16:07:24 +0200 Subject: [PATCH 010/100] Allow installing akkoma-fe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/config.exs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config/config.exs b/config/config.exs index 5d2bf01fa5..535621e0b6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -830,6 +830,21 @@ "https://lily-is.land/infra/glitch-lily/-/jobs/artifacts/${ref}/download?job=build", "ref" => "servant", "build_dir" => "public" + }, + "akkoma-fe" => %{ + "name" => "akkoma-fe", + "git" => "https://akkoma.dev/AkkomaGang/akkoma-fe", + "build_url" => + "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/${ref}/akkoma-fe.zip", + "ref" => "develop", + "build_dir" => "dist" + }, + "akkoma-admin-fe" => %{ + "name" => "akkoma-admin-fe", + "git" => "https://akkoma.dev/AkkomaGang/admin-fe", + "build_url" => + "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/${ref}/admin-fe.zip", + "ref" => "stable" } } From 63a028888549ddba637dc08e186698d8cb6bc069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 8 Sep 2024 16:22:02 +0200 Subject: [PATCH 011/100] expose bubble timeline availability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 8d0171bf3d..b5263fca1b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -183,6 +183,9 @@ def features do "events", "multitenancy", "pleroma:bites", + if !Enum.empty?(Config.get([:instance, :local_bubble], [])) do + "bubble_timeline" + end, # Akkoma compatibility if Pleroma.Language.Translation.configured?() do "akkoma:machine_translation" From 509d5ef74a7b649974b39954fa3ed64cb88b47d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 9 Sep 2024 16:15:45 +0200 Subject: [PATCH 012/100] we're not soapbox actually MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- README.md | 2 +- config/config.exs | 4 ++-- config/{soapbox.exs => pl-fe.exs} | 9 +-------- lib/pleroma/web/plugs/http_security_plug.ex | 4 ++-- lib/pleroma/web/templates/layout/app.html.eex | 10 +++++----- lib/pleroma/web/utils/colors.ex | 2 +- .../controllers/instance_controller_test.exs | 4 ++-- test/pleroma/web/utils/colors_test.exs | 4 ++-- 8 files changed, 16 insertions(+), 23 deletions(-) rename config/{soapbox.exs => pl-fe.exs} (87%) diff --git a/README.md b/README.md index b58ff35eba..302d95e817 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Added features: - [Partial implementation of Mastodon admin API](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3671) - [Moderators can assign users to reports](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3670) - [Mastodon-compatible webhooks](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3683) -- UI is restyled to match pl-fe/Soapbox visuals +- UI is restyled to match pl-fe visuals Features not authored by me: - [Chat deletion](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3029) diff --git a/config/config.exs b/config/config.exs index 535621e0b6..0e921bf6dd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -978,8 +978,6 @@ config :geospatial, Geospatial.HTTP, user_agent: &Pleroma.Application.user_agent/0 -import_config "soapbox.exs" - config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch config :pleroma, Pleroma.Search.Meilisearch, @@ -1009,6 +1007,8 @@ vectors: %{size: 384, distance: "Cosine"} } +import_config "pl-fe.exs" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/soapbox.exs b/config/pl-fe.exs similarity index 87% rename from config/soapbox.exs rename to config/pl-fe.exs index b7f270e7a8..584c1c6d52 100644 --- a/config/soapbox.exs +++ b/config/pl-fe.exs @@ -1,11 +1,8 @@ -# Soapbox default config overrides +# pl-fe default config overrides # This file gets loaded after config.exs # and before prod.secret.exs import Config -# Twitter-like block behavior -config :pleroma, :activitypub, blockers_visible: false - # Sane default upload filters config :pleroma, Pleroma.Upload, filters: [ @@ -45,11 +42,7 @@ # Sane default media attachment limit config :pleroma, :instance, max_media_attachments: 20 -# Use Soapbox branding config :pleroma, :instance, - name: "Soapbox", - description: "Social media owned by you", - instance_thumbnail: "/instance/thumbnail.png", account_approval_required: true, moderator_privileges: [ :users_read, diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 58f38b4b1f..c43f4ecf68 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -92,7 +92,7 @@ defp csp_string do static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() report_uri = @config_impl.get([:http_security, :report_uri]) - sentry_dsn = @config_impl.get([:frontend_configurations, :soapbox_fe, "sentryDsn"]) + sentry_dsn = @config_impl.get([:frontend_configurations, :pl_fe, "sentryDsn"]) img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" @@ -200,7 +200,7 @@ defp build_csp_multimedia_source_list do defp map_tile_server do with tile_server when is_binary(tile_server) <- - @config_impl.get([:frontend_configurations, :soapbox_fe, "tileServer"]), + @config_impl.get([:frontend_configurations, :pl_fe, "tileServer"]), %{host: host} <- URI.parse(tile_server) do ["*.#{host}"] else diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index ac5d9800c1..0e78567e93 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -9,13 +9,13 @@ :root { <%= Pleroma.Web.Utils.Colors.shades_to_css( "primary", - Pleroma.Config.get([:frontend_configurations, :soapbox_fe, "brandColor"], "#0482d8"), - Pleroma.Config.get([:frontend_configurations, :soapbox_fe, "colors", "primary"], %{}) + Pleroma.Config.get([:frontend_configurations, :pl_fe, "brandColor"], "#d80482"), + Pleroma.Config.get([:frontend_configurations, :pl_fe, "colors", "primary"], %{}) ) %> <%= Pleroma.Web.Utils.Colors.shades_to_css( "accent", - Pleroma.Config.get([:frontend_configurations, :soapbox_fe, "accentColor"], "#2bd110"), - Pleroma.Config.get([:frontend_configurations, :soapbox_fe, "colors", "accent"], %{}) + Pleroma.Config.get([:frontend_configurations, :pl_fe, "accentColor"], "#d110b4"), + Pleroma.Config.get([:frontend_configurations, :pl_fe, "colors", "accent"], %{}) ) %> } @@ -23,7 +23,7 @@
- "> + ">

<%= Pleroma.Config.get([:instance, :name]) %>
diff --git a/lib/pleroma/web/utils/colors.ex b/lib/pleroma/web/utils/colors.ex index 449393f2b4..c00a938109 100644 --- a/lib/pleroma/web/utils/colors.ex +++ b/lib/pleroma/web/utils/colors.ex @@ -51,7 +51,7 @@ def get_shades("#" <> base_color, overrides) do shades end - def get_shades(_, overrides), do: get_shades("#0482d8", overrides) + def get_shades(_, overrides), do: get_shades("#d80482", overrides) defp get_override(level, overrides) do if Map.has_key?(overrides, "#{level}") do diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index c6299f0345..bf08cc4be6 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -125,7 +125,7 @@ test "get oauth_consumer_strategies", %{conn: conn} do describe "instance domain blocks" do setup do - clear_config([:mrf_simple, :reject], [{"fediverse.pl", "uses Soapbox"}]) + clear_config([:mrf_simple, :reject], [{"fediverse.pl", "uses pl-fe"}]) end test "get instance domain blocks", %{conn: conn} do @@ -133,7 +133,7 @@ test "get instance domain blocks", %{conn: conn} do assert [ %{ - "comment" => "uses Soapbox", + "comment" => "uses pl-fe", "digest" => "55e3f44aefe7eb022d3b1daaf7396cabf7f181bf6093c8ea841e30c9fc7d8226", "domain" => "fediverse.pl", "severity" => "suspend" diff --git a/test/pleroma/web/utils/colors_test.exs b/test/pleroma/web/utils/colors_test.exs index 91d23f6265..68d4a90e0c 100644 --- a/test/pleroma/web/utils/colors_test.exs +++ b/test/pleroma/web/utils/colors_test.exs @@ -25,9 +25,9 @@ test "generates tints from a base color" do } == Colors.get_shades(@base_color) end - test "uses soapbox blue if invalid color provided" do + test "uses pl-fe default color if invalid color provided" do assert %{ - 500 => "4, 130, 216" + 500 => "216, 4, 130" } = Colors.get_shades("255, 255, 127") end end From 91f08614495d3ced45bbcee1233d699110c9b844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 10 Sep 2024 13:05:49 +0200 Subject: [PATCH 013/100] yes, i'm not a big fan of this `api_versions` thing, how did you know? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index b5263fca1b..ce0588e5b9 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -78,6 +78,11 @@ def render("show2.json", _) do email: Keyword.get(instance, :email), account: contact_account(Keyword.get(instance, :contact_username)) }, + api_versions: %{ + "mastodon" => 2137, + "social.pleroma" => 420, + "pl.mkljczk.pl" => 69 + }, # Extra (not present in Mastodon): pleroma: pleroma_configuration2(instance) }) From 23f548dc614515800589799eeda93aae8d8ae11f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 10 Sep 2024 22:28:50 +0200 Subject: [PATCH 014/100] Akkoma migration: Move mastofe settings to settings store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../20220108213213_add_mastofe_settings.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs b/priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs index 7f98807fb6..ac22ae6594 100644 --- a/priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs +++ b/priv/repo/optional_migrations/akkoma_rollbacks/20220108213213_add_mastofe_settings.exs @@ -7,6 +7,16 @@ defmodule Pleroma.Repo.Migrations.AddMastofeSettings do def up, do: :ok def down do + """ + UPDATE users SET pleroma_settings_store = jsonb_set( + pleroma_settings_store, + '{glitch-lily}', + mastofe_settings + ) + WHERE mastofe_settings IS NOT NULL; + """ + |> execute() + alter table(:users) do remove_if_exists(:mastofe_settings, :map) end From 7def11d7c352f13ce0f12715649359344cbba9a6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 11 Sep 2024 12:45:33 -0400 Subject: [PATCH 015/100] LDAP Auth: fix TLS certificate verification Currently we only support STARTTLS and it was not verifying certificate and hostname correctly. We must pass a custom fqdn_fun/1 function so it knows what value to compare against. --- changelog.d/ldap-tls.fix | 1 + lib/pleroma/web/auth/ldap_authenticator.ex | 12 +++++++++++- mix.exs | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ldap-tls.fix diff --git a/changelog.d/ldap-tls.fix b/changelog.d/ldap-tls.fix new file mode 100644 index 0000000000..b15137d775 --- /dev/null +++ b/changelog.d/ldap-tls.fix @@ -0,0 +1 @@ +STARTTLS certificate and hostname verification for LDAP authentication diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index ea5620cf60..d31f347479 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -41,6 +41,7 @@ defp ldap_user(name, password) do port = Keyword.get(ldap, :port, 389) ssl = Keyword.get(ldap, :ssl, false) sslopts = Keyword.get(ldap, :sslopts, []) + tlsopts = Keyword.get(ldap, :tlsopts, []) options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] ++ @@ -54,7 +55,16 @@ defp ldap_user(name, password) do case :eldap.start_tls( connection, - Keyword.get(ldap, :tlsopts, []), + Keyword.merge( + [ + verify: :verify_peer, + cacerts: :certifi.cacerts(), + customize_hostname_check: [ + fqdn_fun: fn _ -> to_charlist(host) end + ] + ], + tlsopts + ), @connection_timeout ) do :ok -> diff --git a/mix.exs b/mix.exs index 0d49a6b45f..9a261547f3 100644 --- a/mix.exs +++ b/mix.exs @@ -204,6 +204,7 @@ defp deps do {:oban_live_dashboard, "~> 0.1.1"}, {:multipart, "~> 0.4.0", optional: true}, {:argon2_elixir, "~> 4.0"}, + {:certifi, "~> 2.12"}, ## dev & test {:phoenix_live_reload, "~> 1.3.3", only: :dev}, From 291061032c90caabff44d54969baa523cda3f2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 00:11:15 +0200 Subject: [PATCH 016/100] update test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/web/utils/colors_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/utils/colors_test.exs b/test/pleroma/web/utils/colors_test.exs index 68d4a90e0c..98c5d0745e 100644 --- a/test/pleroma/web/utils/colors_test.exs +++ b/test/pleroma/web/utils/colors_test.exs @@ -34,6 +34,6 @@ test "uses pl-fe default color if invalid color provided" do test "shades_to_css/2" do result = Colors.shades_to_css("primary") - assert String.contains?(result, "--color-primary-500: 4, 130, 216;") + assert String.contains?(result, "--color-primary-500: 216, 4, 130;") end end From 7a2200faa8bdd7d8921b79aea9422e4daf6c25fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 11:17:45 +0200 Subject: [PATCH 017/100] Include Pleroma logo in .svg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- priv/static/static/logo.svg | 71 +++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 priv/static/static/logo.svg diff --git a/priv/static/static/logo.svg b/priv/static/static/logo.svg new file mode 100644 index 0000000000..68e647e6ca --- /dev/null +++ b/priv/static/static/logo.svg @@ -0,0 +1,71 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + From 106abc34b1fd0f2c6d1aa9915414c890935c1727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 11:26:26 +0200 Subject: [PATCH 018/100] declare support for window controls overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/views/manifest_view.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/views/manifest_view.ex b/lib/pleroma/web/views/manifest_view.ex index f8adf0f58f..0e7a34adfa 100644 --- a/lib/pleroma/web/views/manifest_view.ex +++ b/lib/pleroma/web/views/manifest_view.ex @@ -15,6 +15,7 @@ def render("manifest.json", _params) do theme_color: Config.get([:manifest, :theme_color]), background_color: Config.get([:manifest, :background_color]), display: "standalone", + "display_override": ["window-controls-overlay"], scope: Endpoint.url(), start_url: "/", categories: [ From affdcdb68daabb15f8fad2e7b6406606e8086e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 11:27:29 +0200 Subject: [PATCH 019/100] Manifest: declare /static/logo.svg as 512x512 to match one provided by pleroma-fe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/manifest-icon-size.skip | 0 config/config.exs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/manifest-icon-size.skip diff --git a/changelog.d/manifest-icon-size.skip b/changelog.d/manifest-icon-size.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/config.exs b/config/config.exs index 80a3b8d57f..cd9a2539fa 100644 --- a/config/config.exs +++ b/config/config.exs @@ -344,7 +344,7 @@ icons: [ %{ src: "/static/logo.svg", - sizes: "144x144", + sizes: "512x512", purpose: "any", type: "image/svg+xml" } From a6ec71873a2f2ad9c19789e2781c925048ce0c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 11:30:37 +0200 Subject: [PATCH 020/100] Manifest: prefer theme color provided by pl fe configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/views/manifest_view.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/views/manifest_view.ex b/lib/pleroma/web/views/manifest_view.ex index 0e7a34adfa..14f6ba5455 100644 --- a/lib/pleroma/web/views/manifest_view.ex +++ b/lib/pleroma/web/views/manifest_view.ex @@ -12,10 +12,14 @@ def render("manifest.json", _params) do name: Config.get([:instance, :name]), description: Config.get([:instance, :description]), icons: Config.get([:manifest, :icons]), - theme_color: Config.get([:manifest, :theme_color]), + theme_color: + Config.get( + [:frontend_configurations, :pl_fe, "brandColor"], + Config.get([:manifest, :theme_color]) + ), background_color: Config.get([:manifest, :background_color]), display: "standalone", - "display_override": ["window-controls-overlay"], + display_override: ["window-controls-overlay"], scope: Endpoint.url(), start_url: "/", categories: [ From 17b69c43d5ed6ba867f5fb1da15f6af9aa7c5d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 14:37:37 +0200 Subject: [PATCH 021/100] Add `group_key` to notifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/notifications-group-key.add | 1 + .../api_spec/operations/notification_operation.ex | 5 +++++ .../web/mastodon_api/views/notification_view.ex | 1 + .../mastodon_api/views/notification_view_test.exs | 13 +++++++++++++ 4 files changed, 20 insertions(+) create mode 100644 changelog.d/notifications-group-key.add diff --git a/changelog.d/notifications-group-key.add b/changelog.d/notifications-group-key.add new file mode 100644 index 0000000000..386927f4a4 --- /dev/null +++ b/changelog.d/notifications-group-key.add @@ -0,0 +1 @@ +Add `group_key` to notifications \ No newline at end of file diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 2dc0f66df5..94d1f6b82a 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -158,6 +158,10 @@ def notification do type: :object, properties: %{ id: %Schema{type: :string}, + group_key: %Schema{ + type: :string, + description: "Group key shared by similar notifications" + }, type: notification_type(), created_at: %Schema{type: :string, format: :"date-time"}, account: %Schema{ @@ -180,6 +184,7 @@ def notification do }, example: %{ "id" => "34975861", + "group-key" => "ungrouped-34975861", "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 3f24787192..c277af98b5 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -95,6 +95,7 @@ def render( response = %{ id: to_string(notification.id), + group_key: "ungrouped-" <> to_string(notification.id), type: notification.type, created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 75ab375aae..b1f3523acb 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -56,6 +56,7 @@ test "ChatMessage notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:chat_mention", account: AccountView.render("show.json", %{user: user, for: recipient}), @@ -75,6 +76,7 @@ test "Mention notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "mention", account: @@ -99,6 +101,7 @@ test "Favourite notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -119,6 +122,7 @@ test "Reblog notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "reblog", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -137,6 +141,7 @@ test "Follow notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "follow", account: AccountView.render("show.json", %{user: follower, for: followed}), @@ -165,6 +170,7 @@ test "Move notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "move", account: AccountView.render("show.json", %{user: old_user, for: follower}), @@ -190,6 +196,7 @@ test "EmojiReact notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:emoji_reaction", emoji: "☕", @@ -229,6 +236,7 @@ test "EmojiReact custom emoji notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:emoji_reaction", emoji: ":dinosaur:", @@ -248,6 +256,7 @@ test "Poll notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "poll", account: @@ -274,6 +283,7 @@ test "Report notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:report", account: AccountView.render("show.json", %{user: reporting_user, for: moderator_user}), @@ -300,6 +310,7 @@ test "Edit notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "update", account: AccountView.render("show.json", %{user: user, for: repeat_user}), @@ -322,6 +333,7 @@ test "muted notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: true, is_muted: true}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -345,6 +357,7 @@ test "Subscribed status notification" do expected = %{ id: to_string(notification.id), + group_key: "ungrouped-#{to_string(notification.id)}", pleroma: %{is_seen: false, is_muted: false}, type: "status", account: From f813da39ce6d34200e7b9028ff4640807a76cd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 15:45:56 +0200 Subject: [PATCH 022/100] fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .github/workflows/pl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pl.yaml b/.github/workflows/pl.yaml index e66ccea358..58cbb980db 100644 --- a/.github/workflows/pl.yaml +++ b/.github/workflows/pl.yaml @@ -1,4 +1,4 @@ -# Adapter from https://fly.io/phoenix-files/github-actions-for-elixir-ci/ +# Adapted from https://fly.io/phoenix-files/github-actions-for-elixir-ci/ name: pl CI From e10db52e0a1c9cc24803a406998a9cfe75b7f9f2 Mon Sep 17 00:00:00 2001 From: Mint Date: Fri, 13 Sep 2024 02:58:59 +0300 Subject: [PATCH 023/100] Add dependencies for Swoosh's Mua mail adapter --- changelog.d/swoosh-mua.add | 1 + mix.exs | 4 +++- mix.lock | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changelog.d/swoosh-mua.add diff --git a/changelog.d/swoosh-mua.add b/changelog.d/swoosh-mua.add new file mode 100644 index 0000000000..d4c4bbd084 --- /dev/null +++ b/changelog.d/swoosh-mua.add @@ -0,0 +1 @@ +Added dependencies for Swoosh's Mua mail adapter diff --git a/mix.exs b/mix.exs index 0d49a6b45f..ceae5c26df 100644 --- a/mix.exs +++ b/mix.exs @@ -153,7 +153,7 @@ defp deps do {:calendar, "~> 1.0"}, {:cachex, "~> 3.2"}, {:tesla, "~> 1.11"}, - {:castore, "~> 0.1"}, + {:castore, "~> 1.0"}, {:cowlib, "~> 2.9", override: true}, {:gun, "~> 2.0.0-rc.1", override: true}, {:finch, "~> 0.15"}, @@ -169,6 +169,8 @@ defp deps do {:swoosh, "~> 1.16.9"}, {:phoenix_swoosh, "~> 1.1"}, {:gen_smtp, "~> 0.13"}, + {:mua, "~> 0.2.0"}, + {:mail, "~> 0.3.0"}, {:ex_syslogger, "~> 1.4"}, {:floki, "~> 0.35"}, {:timex, "~> 3.6"}, diff --git a/mix.lock b/mix.lock index 01f2eef98e..2cf44862b4 100644 --- a/mix.lock +++ b/mix.lock @@ -11,7 +11,7 @@ "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, "calendar": {:hex, :calendar, "1.0.0", "f52073a708528482ec33d0a171954ca610fe2bd28f1e871f247dc7f1565fa807", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "990e9581920c82912a5ee50e62ff5ef96da6b15949a2ee4734f935fdef0f0a6f"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "6630c42aaaab124e697b4e513190c89d8b64e410", [ref: "6630c42aaaab124e697b4e513190c89d8b64e410"]}, - "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"}, + "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, "cc_precompiler": {:hex, :cc_precompiler, "0.1.9", "e8d3364f310da6ce6463c3dd20cf90ae7bbecbf6c5203b98bf9b48035592649b", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9dcab3d0f3038621f1601f13539e7a9ee99843862e66ad62827b0c42b2f58a54"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, @@ -72,6 +72,7 @@ "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, "linkify": {:hex, :linkify, "0.5.3", "5f8143d8f61f5ff08d3aeeff47ef6509492b4948d8f08007fbf66e4d2246a7f2", [:mix], [], "hexpm", "3ef35a1377d47c25506e07c1c005ea9d38d700699d92ee92825f024434258177"}, "logger_backends": {:hex, :logger_backends, "1.0.0", "09c4fad6202e08cb0fbd37f328282f16539aca380f512523ce9472b28edc6bdf", [:mix], [], "hexpm", "1faceb3e7ec3ef66a8f5746c5afd020e63996df6fd4eb8cdb789e5665ae6c9ce"}, + "mail": {:hex, :mail, "0.3.1", "cb0a14e4ed8904e4e5a08214e686ccf6f9099346885db17d8c309381f865cc5c", [:mix], [], "hexpm", "1db701e89865c1d5fa296b2b57b1cd587587cca8d8a1a22892b35ef5a8e352a6"}, "majic": {:hex, :majic, "1.0.0", "37e50648db5f5c2ff0c9fb46454d034d11596c03683807b9fb3850676ffdaab3", [:make, :mix], [{:elixir_make, "~> 0.6.1", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "7905858f76650d49695f14ea55cd9aaaee0c6654fa391671d4cf305c275a0a9e"}, "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"}, @@ -85,6 +86,7 @@ "mock": {:hex, :mock, "0.3.8", "7046a306b71db2488ef54395eeb74df0a7f335a7caca4a3d3875d1fc81c884dd", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "7fa82364c97617d79bb7d15571193fc0c4fe5afd0c932cef09426b3ee6fe2022"}, "mogrify": {:hex, :mogrify, "0.9.3", "238c782f00271dace01369ad35ae2e9dd020feee3443b9299ea5ea6bed559841", [:mix], [], "hexpm", "0189b1e1de27455f2b9ae8cf88239cefd23d38de9276eb5add7159aea51731e6"}, "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, + "mua": {:hex, :mua, "0.2.3", "46b29b7b2bb14105c0b7be9526f7c452df17a7841b30b69871c024a822ff551c", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "7fe861a87fcc06a980d3941bbcb2634e5f0f30fd6ad15ef6c0423ff9dc7e46de"}, "multipart": {:hex, :multipart, "0.4.0", "634880a2148d4555d050963373d0e3bbb44a55b2badd87fa8623166172e9cda0", [:mix], [{:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "3c5604bc2fb17b3137e5d2abdf5dacc2647e60c5cc6634b102cf1aef75a06f0a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, From 26fc74835f9c91115bacad5c59902f9e91c8ceae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 14 Sep 2024 20:17:08 +0200 Subject: [PATCH 024/100] federate avatar/header descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 5 +++ .../web/activity_pub/views/user_view.ex | 36 +++++++++++++++---- .../web/mastodon_api/views/account_view.ex | 8 ++--- .../web/activity_pub/views/user_view_test.exs | 17 +++++++++ 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 5170092534..7a36ece779 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -419,6 +419,11 @@ def banner_url(user, options \\ []) do end end + def image_description(image, default \\ "") + + def image_description(%{"name" => name}, _default), do: name + def image_description(_, default), do: default + # Should probably be renamed or removed @spec ap_id(User.t()) :: String.t() def ap_id(%User{nickname: nickname}), do: "#{Endpoint.url()}/users/#{nickname}" diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 937e4fd67d..cd485ed64a 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -129,8 +129,22 @@ def render("user.json", %{user: user}) do "vcard:bday" => birthday, "webfinger" => "acct:#{User.full_nickname(user)}" } - |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) - |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) + |> Map.merge( + maybe_make_image( + &User.avatar_url/2, + User.image_description(user.avatar, nil), + "icon", + user + ) + ) + |> Map.merge( + maybe_make_image( + &User.banner_url/2, + User.image_description(user.banner, nil), + "image", + user + ) + ) |> Map.merge(Utils.make_json_ld_header()) end @@ -305,16 +319,24 @@ def collection(collection, iri, page, show_items \\ true, total \\ nil) do end end - defp maybe_make_image(func, key, user) do + defp maybe_make_image(func, description, key, user) do if image = func.(user, no_default: true) do %{ - key => %{ - "type" => "Image", - "url" => image - } + key => + %{ + "type" => "Image", + "url" => image + } + |> maybe_put_description(description) } else %{} end end + + defp maybe_put_description(map, description) when is_binary(description) do + Map.put(map, "name", description) + end + + defp maybe_put_description(map, _description), do: map end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 7de6745d47..f6727d29d6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -219,10 +219,10 @@ defp do_render("show.json", %{user: user} = opts) do avatar = User.avatar_url(user) |> MediaProxy.url() avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(static: true) - avatar_description = image_description(user.avatar) + avatar_description = User.image_description(user.avatar) header = User.banner_url(user) |> MediaProxy.url() header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true) - header_description = image_description(user.banner) + header_description = User.image_description(user.banner) following_count = if !user.hide_follows_count or !user.hide_follows or self, @@ -349,10 +349,6 @@ defp username_from_nickname(string) when is_binary(string) do defp username_from_nickname(_), do: nil - defp image_description(%{"name" => name}), do: name - - defp image_description(_), do: "" - defp maybe_put_follow_requests_count( data, %User{id: user_id} = user, diff --git a/test/pleroma/web/activity_pub/views/user_view_test.exs b/test/pleroma/web/activity_pub/views/user_view_test.exs index 651e535ac4..1b11a5f41e 100644 --- a/test/pleroma/web/activity_pub/views/user_view_test.exs +++ b/test/pleroma/web/activity_pub/views/user_view_test.exs @@ -68,6 +68,23 @@ test "Does not add an avatar image if the user hasn't set one" do result = UserView.render("user.json", %{user: user}) assert result["icon"]["url"] == "https://someurl" assert result["image"]["url"] == "https://somebanner" + + refute result["icon"]["name"] + refute result["image"]["name"] + end + + test "Avatar has a description if the user set one" do + user = + insert(:user, + avatar: %{ + "url" => [%{"href" => "https://someurl"}], + "name" => "pleroma-tan using pleroma groups" + } + ) + + result = UserView.render("user.json", %{user: user}) + + assert result["icon"]["name"] == "pleroma-tan using pleroma groups" end test "renders an invisible user with the invisible property set to true" do From 455e3dea2e57ef0b28ac685ff6600b3d0c89c064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 12 Sep 2024 14:24:13 +0200 Subject: [PATCH 025/100] Use `admin.report` for report notification type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../admin-report-notification-type.change | 1 + .../API/differences_in_mastoapi_responses.md | 11 +-- lib/pleroma/notification.ex | 5 +- .../operations/notification_operation.ex | 2 - lib/pleroma/web/mastodon_api/mastodon_api.ex | 6 +- .../mastodon_api/views/notification_view.ex | 2 +- ...port_notification_type_to_admin_report.exs | 91 +++++++++++++++++++ test/pleroma/notification_test.exs | 4 +- .../notification_controller_test.exs | 10 +- .../views/notification_view_test.exs | 2 +- 10 files changed, 108 insertions(+), 26 deletions(-) create mode 100644 changelog.d/admin-report-notification-type.change create mode 100644 priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs diff --git a/changelog.d/admin-report-notification-type.change b/changelog.d/admin-report-notification-type.change new file mode 100644 index 0000000000..6899a131fd --- /dev/null +++ b/changelog.d/admin-report-notification-type.change @@ -0,0 +1 @@ +Use `admin.report` for report notification type \ No newline at end of file diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 41464e8021..dfe7b5bf63 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -181,21 +181,12 @@ The `type` value is `pleroma:chat_mention` - `account`: The account who sent the message - `chat_message`: The chat message -### Report Notification (not default) - -This notification has to be requested explicitly. - -The `type` value is `pleroma:report` - -- `account`: The account who reported -- `report`: The report - ## GET `/api/v1/notifications` Accepts additional parameters: - `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`. -- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`, `pleroma:chat_mention`, `pleroma:report`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`. +- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`, `pleroma:chat_mention`, `admin.report`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`. ## DELETE `/api/v1/notifications/destroy_multiple` diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 75f4ba5033..3b5b6fb947 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -70,10 +70,11 @@ def unread_notifications_count(%User{id: user_id}) do move pleroma:chat_mention pleroma:emoji_reaction - pleroma:report + admin.report reblog poll status + update } def changeset(%Notification{} = notification, attrs) do @@ -412,7 +413,7 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do "pleroma:emoji_reaction" "Flag" -> - "pleroma:report" + "admin.report" # Compatibility with old reactions "EmojiReaction" -> diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 2dc0f66df5..f5fc85e9ca 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -199,7 +199,6 @@ defp notification_type do "mention", "pleroma:emoji_reaction", "pleroma:chat_mention", - "pleroma:report", "move", "follow_request", "poll", @@ -219,7 +218,6 @@ defp notification_type do - `move` - Someone moved their account - `pleroma:emoji_reaction` - Someone reacted with emoji to your status - `pleroma:chat_mention` - Someone mentioned you in a chat message - - `pleroma:report` - Someone was reported - `status` - Someone you are subscribed to created a status - `update` - A status you boosted has been edited - `admin.sign_up` - Someone signed up (optionally sent to admins) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index c9e045d238..9e049fcdcd 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -65,14 +65,14 @@ def get_notifications(user, params \\ %{}) do cast_params(params) |> Map.update(:include_types, [], fn include_types -> include_types end) options = - if ("pleroma:report" not in options.include_types and + if ("admin.report" not in options.include_types and User.privileged?(user, :reports_manage_reports)) or User.privileged?(user, :reports_manage_reports) do options else options - |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> - current_exclude_types ++ ["pleroma:report"] + |> Map.update(:exclude_types, ["admin.report"], fn current_exclude_types -> + current_exclude_types ++ ["admin.report"] end) end diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 3f24787192..bfb8020282 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -134,7 +134,7 @@ def render( "pleroma:chat_mention" -> put_chat_message(response, activity, reading_user, status_render_opts) - "pleroma:report" -> + "admin.report" -> put_report(response, activity) type when type in ["follow", "follow_request"] -> diff --git a/priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs b/priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs new file mode 100644 index 0000000000..df9f6343a5 --- /dev/null +++ b/priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs @@ -0,0 +1,91 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.RenamePleromaReportNotificationTypeToAdminReport do + use Ecto.Migration + + def up do + alter table(:notifications) do + modify(:type, :string) + end + + """ + update notifications + set type = 'admin.report' + where type = 'pleroma:report' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'admin.report', + 'poll', + 'status', + 'update' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end + + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + update notifications + set type = 'pleroma:report' + where type = 'admin.report' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll', + 'status', + 'update' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index e595c5c532..c2cfd3dc0f 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -48,7 +48,7 @@ test "creates a report notification only for privileged users" do {:ok, [notification]} = Notification.create_notifications(activity2) assert notification.user_id == moderator_user.id - assert notification.type == "pleroma:report" + assert notification.type == "admin.report" end test "suppresses notifications for own reports" do @@ -64,7 +64,7 @@ test "suppresses notifications for own reports" do refute notification.user_id == reporting_admin.id assert notification.user_id == other_admin.id - assert notification.type == "pleroma:report" + assert notification.type == "admin.report" end test "creates a notification for an emoji reaction" do diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index 8fc22dde14..29eddddb1d 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -78,7 +78,7 @@ test "by default, does not contain pleroma:chat_mention" do assert [_] = result end - test "by default, does not contain pleroma:report" do + test "by default, does not contain admin.report" do clear_config([:instance, :moderator_privileges], [:reports_manage_reports]) user = insert(:user) @@ -103,13 +103,13 @@ test "by default, does not contain pleroma:report" do result = conn - |> get("/api/v1/notifications?include_types[]=pleroma:report") + |> get("/api/v1/notifications?include_types[]=admin.report") |> json_response_and_validate_schema(200) assert [_] = result end - test "Pleroma:report is hidden for non-privileged users" do + test "Admin.report is hidden for non-privileged users" do clear_config([:instance, :moderator_privileges], [:reports_manage_reports]) user = insert(:user) @@ -127,7 +127,7 @@ test "Pleroma:report is hidden for non-privileged users" do result = conn - |> get("/api/v1/notifications?include_types[]=pleroma:report") + |> get("/api/v1/notifications?include_types[]=admin.report") |> json_response_and_validate_schema(200) assert [_] = result @@ -136,7 +136,7 @@ test "Pleroma:report is hidden for non-privileged users" do result = conn - |> get("/api/v1/notifications?include_types[]=pleroma:report") + |> get("/api/v1/notifications?include_types[]=admin.report") |> json_response_and_validate_schema(200) assert [] == result diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 75ab375aae..1439f1c350 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -275,7 +275,7 @@ test "Report notification" do expected = %{ id: to_string(notification.id), pleroma: %{is_seen: false, is_muted: false}, - type: "pleroma:report", + type: "admin.report", account: AccountView.render("show.json", %{user: reporting_user, for: moderator_user}), created_at: Utils.to_masto_date(notification.inserted_at), report: ReportView.render("show.json", Report.extract_report_info(activity)) From 1a120d013019fc15b3f440f7db71d3eb328bc798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 14 Sep 2024 20:17:08 +0200 Subject: [PATCH 026/100] Federate avatar/header descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/profile-image-descriptions.skip | 0 lib/pleroma/user.ex | 5 +++ lib/pleroma/web/activity_pub/activity_pub.ex | 9 ++++- .../web/activity_pub/views/user_view.ex | 36 +++++++++++++++---- .../web/mastodon_api/views/account_view.ex | 8 ++--- .../web/activity_pub/activity_pub_test.exs | 35 ++++++++++++++++-- .../web/activity_pub/views/user_view_test.exs | 17 +++++++++ 7 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 changelog.d/profile-image-descriptions.skip diff --git a/changelog.d/profile-image-descriptions.skip b/changelog.d/profile-image-descriptions.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 5170092534..7a36ece779 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -419,6 +419,11 @@ def banner_url(user, options \\ []) do end end + def image_description(image, default \\ "") + + def image_description(%{"name" => name}, _default), do: name + def image_description(_, default), do: default + # Should probably be renamed or removed @spec ap_id(User.t()) :: String.t() def ap_id(%User{nickname: nickname}), do: "#{Endpoint.url()}/users/#{nickname}" diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a2a94a0ff4..df8795fe4b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1542,16 +1542,23 @@ defp get_actor_url(url) when is_list(url) do defp get_actor_url(_url), do: nil - defp normalize_image(%{"url" => url}) do + defp normalize_image(%{"url" => url} = data) do %{ "type" => "Image", "url" => [%{"href" => url}] } + |> maybe_put_description(data) end defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image() defp normalize_image(_), do: nil + defp maybe_put_description(map, %{"name" => description}) when is_binary(description) do + Map.put(map, "name", description) + end + + defp maybe_put_description(map, _), do: map + defp object_to_user_data(data, additional) do fields = data diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 937e4fd67d..cd485ed64a 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -129,8 +129,22 @@ def render("user.json", %{user: user}) do "vcard:bday" => birthday, "webfinger" => "acct:#{User.full_nickname(user)}" } - |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) - |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) + |> Map.merge( + maybe_make_image( + &User.avatar_url/2, + User.image_description(user.avatar, nil), + "icon", + user + ) + ) + |> Map.merge( + maybe_make_image( + &User.banner_url/2, + User.image_description(user.banner, nil), + "image", + user + ) + ) |> Map.merge(Utils.make_json_ld_header()) end @@ -305,16 +319,24 @@ def collection(collection, iri, page, show_items \\ true, total \\ nil) do end end - defp maybe_make_image(func, key, user) do + defp maybe_make_image(func, description, key, user) do if image = func.(user, no_default: true) do %{ - key => %{ - "type" => "Image", - "url" => image - } + key => + %{ + "type" => "Image", + "url" => image + } + |> maybe_put_description(description) } else %{} end end + + defp maybe_put_description(map, description) when is_binary(description) do + Map.put(map, "name", description) + end + + defp maybe_put_description(map, _description), do: map end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 7de6745d47..f6727d29d6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -219,10 +219,10 @@ defp do_render("show.json", %{user: user} = opts) do avatar = User.avatar_url(user) |> MediaProxy.url() avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(static: true) - avatar_description = image_description(user.avatar) + avatar_description = User.image_description(user.avatar) header = User.banner_url(user) |> MediaProxy.url() header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true) - header_description = image_description(user.banner) + header_description = User.image_description(user.banner) following_count = if !user.hide_follows_count or !user.hide_follows or self, @@ -349,10 +349,6 @@ defp username_from_nickname(string) when is_binary(string) do defp username_from_nickname(_), do: nil - defp image_description(%{"name" => name}), do: name - - defp image_description(_), do: "" - defp maybe_put_follow_requests_count( data, %User{id: user_id} = user, diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index b4f6fb68ae..72222ae889 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -232,12 +232,14 @@ test "works for bridgy actors" do assert user.avatar == %{ "type" => "Image", - "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}] + "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}], + "name" => "profile picture" } assert user.banner == %{ "type" => "Image", - "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}] + "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}], + "name" => "profile picture" } end @@ -432,6 +434,35 @@ test "fetches user birthday information from misskey" do assert user.birthday == ~D[2001-02-12] end + + test "fetches avatar description" do + user_id = "https://example.com/users/marcin" + + user_data = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", "marcin") + |> Jason.decode!() + |> Map.delete("featured") + |> Map.update("icon", %{}, fn image -> Map.put(image, "name", "image description") end) + |> Jason.encode!() + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^user_id + } -> + %Tesla.Env{ + status: 200, + body: user_data, + headers: [{"content-type", "application/activity+json"}] + } + end) + + {:ok, user} = ActivityPub.make_user_from_ap_id(user_id) + + assert user.avatar["name"] == "image description" + end end test "it fetches the appropriate tag-restricted posts" do diff --git a/test/pleroma/web/activity_pub/views/user_view_test.exs b/test/pleroma/web/activity_pub/views/user_view_test.exs index 651e535ac4..a32e728296 100644 --- a/test/pleroma/web/activity_pub/views/user_view_test.exs +++ b/test/pleroma/web/activity_pub/views/user_view_test.exs @@ -68,6 +68,23 @@ test "Does not add an avatar image if the user hasn't set one" do result = UserView.render("user.json", %{user: user}) assert result["icon"]["url"] == "https://someurl" assert result["image"]["url"] == "https://somebanner" + + refute result["icon"]["name"] + refute result["image"]["name"] + end + + test "Avatar has a description if the user set one" do + user = + insert(:user, + avatar: %{ + "url" => [%{"href" => "https://someurl"}], + "name" => "a drawing of pleroma-tan using pleroma groups" + } + ) + + result = UserView.render("user.json", %{user: user}) + + assert result["icon"]["name"] == "a drawing of pleroma-tan using pleroma groups" end test "renders an invisible user with the invisible property set to true" do From 5539fea3bb0d272b4cefc2b72755cb3cd285cc67 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 14 Sep 2024 20:03:26 -0400 Subject: [PATCH 027/100] LDAP: permit overriding the CA root --- changelog.d/ldap-ca.add | 1 + config/config.exs | 4 +++- docs/configuration/cheatsheet.md | 1 + lib/pleroma/web/auth/ldap_authenticator.ex | 17 ++++++++++++++++- mix.exs | 1 - 5 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 changelog.d/ldap-ca.add diff --git a/changelog.d/ldap-ca.add b/changelog.d/ldap-ca.add new file mode 100644 index 0000000000..32ecbb5c02 --- /dev/null +++ b/changelog.d/ldap-ca.add @@ -0,0 +1 @@ +LDAP configuration now permits overriding the CA root certificate file for TLS validation. diff --git a/config/config.exs b/config/config.exs index 80a3b8d57f..237928503f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -619,7 +619,9 @@ tls: System.get_env("LDAP_TLS") == "true", tlsopts: [], base: System.get_env("LDAP_BASE") || "dc=example,dc=com", - uid: System.get_env("LDAP_UID") || "cn" + uid: System.get_env("LDAP_UID") || "cn", + # defaults to CAStore's Mozilla roots + cacertfile: nil oauth_consumer_strategies = System.get_env("OAUTH_CONSUMER_STRATEGIES") diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 0b4e53b6f5..4cbde696ee 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -974,6 +974,7 @@ Pleroma account will be created with the same name as the LDAP user name. * `tlsopts`: additional TLS options * `base`: LDAP base, e.g. "dc=example,dc=com" * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" +* `cacertfile`: Path to alternate CA root certificates file Note, if your LDAP server is an Active Directory server the correct value is commonly `uid: "cn"`, but if you use an OpenLDAP server the value may be `uid: "uid"`. diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index d31f347479..7f2cd3d692 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -42,11 +42,14 @@ defp ldap_user(name, password) do ssl = Keyword.get(ldap, :ssl, false) sslopts = Keyword.get(ldap, :sslopts, []) tlsopts = Keyword.get(ldap, :tlsopts, []) + cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] ++ if sslopts != [], do: [{:sslopts, sslopts}], else: [] + cacerts = decode_certfile(cacertfile) + case :eldap.open([to_charlist(host)], options) do {:ok, connection} -> try do @@ -58,7 +61,7 @@ defp ldap_user(name, password) do Keyword.merge( [ verify: :verify_peer, - cacerts: :certifi.cacerts(), + cacerts: cacerts, customize_hostname_check: [ fqdn_fun: fn _ -> to_charlist(host) end ] @@ -147,4 +150,16 @@ defp try_register(name, attributes) do error -> error end end + + defp decode_certfile(file) do + with {:ok, data} <- File.read(file) do + data + |> :public_key.pem_decode() + |> Enum.map(fn {_, b, _} -> b end) + else + _ -> + Logger.error("Unable to read certfile: #{file}") + [] + end + end end diff --git a/mix.exs b/mix.exs index 9a261547f3..0d49a6b45f 100644 --- a/mix.exs +++ b/mix.exs @@ -204,7 +204,6 @@ defp deps do {:oban_live_dashboard, "~> 0.1.1"}, {:multipart, "~> 0.4.0", optional: true}, {:argon2_elixir, "~> 4.0"}, - {:certifi, "~> 2.12"}, ## dev & test {:phoenix_live_reload, "~> 1.3.3", only: :dev}, From af3bf8a4628c0b2981d69f624e3be298adc7dfe6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 15 Sep 2024 13:56:16 -0400 Subject: [PATCH 028/100] Support implicit TLS connections Update docs to clarify that the :ssl option is also for modern TLS, but the :tls option is only for STARTTLS These options may benefit from being renamed but they match upstream terminology. --- changelog.d/ldaps.fix | 1 + docs/configuration/cheatsheet.md | 4 +- lib/pleroma/web/auth/ldap_authenticator.ex | 50 ++++++++++++---------- 3 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 changelog.d/ldaps.fix diff --git a/changelog.d/ldaps.fix b/changelog.d/ldaps.fix new file mode 100644 index 0000000000..a1dc901ab0 --- /dev/null +++ b/changelog.d/ldaps.fix @@ -0,0 +1 @@ +LDAPS connections (implicit TLS) are now supported. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 4cbde696ee..6a535e054f 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -968,9 +968,9 @@ Pleroma account will be created with the same name as the LDAP user name. * `enabled`: enables LDAP authentication * `host`: LDAP server hostname * `port`: LDAP port, e.g. 389 or 636 -* `ssl`: true to use SSL, usually implies the port 636 +* `ssl`: true to use implicit SSL/TLS, usually port 636 * `sslopts`: additional SSL options -* `tls`: true to start TLS, usually implies the port 389 +* `tls`: true to use explicit TLS (STARTTLS), usually port 389 * `tlsopts`: additional TLS options * `base`: LDAP base, e.g. "dc=example,dc=com" * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 7f2cd3d692..18a4e81ee4 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -40,34 +40,39 @@ defp ldap_user(name, password) do host = Keyword.get(ldap, :host, "localhost") port = Keyword.get(ldap, :port, 389) ssl = Keyword.get(ldap, :ssl, false) - sslopts = Keyword.get(ldap, :sslopts, []) - tlsopts = Keyword.get(ldap, :tlsopts, []) + tls = Keyword.get(ldap, :tls, false) cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() - options = - [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] ++ - if sslopts != [], do: [{:sslopts, sslopts}], else: [] + default_secure_opts = [ + verify: :verify_peer, + cacerts: decode_certfile(cacertfile), + customize_hostname_check: [ + fqdn_fun: fn _ -> to_charlist(host) end + ] + ] - cacerts = decode_certfile(cacertfile) + sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, [])) + tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, [])) + + # :sslopts can only be included in :eldap.open/2 when {ssl: true} + # or the connection will fail + options = + if ssl do + [{:port, port}, {:ssl, ssl}, {:sslopts, sslopts}, {:timeout, @connection_timeout}] + else + [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] + end case :eldap.open([to_charlist(host)], options) do {:ok, connection} -> - try do - if Keyword.get(ldap, :tls, false) do + cond do + ssl -> :application.ensure_all_started(:ssl) + tls -> case :eldap.start_tls( connection, - Keyword.merge( - [ - verify: :verify_peer, - cacerts: cacerts, - customize_hostname_check: [ - fqdn_fun: fn _ -> to_charlist(host) end - ] - ], - tlsopts - ), + tlsopts, @connection_timeout ) do :ok -> @@ -75,14 +80,15 @@ defp ldap_user(name, password) do error -> Logger.error("Could not start TLS: #{inspect(error)}") + :eldap.close(connection) end - end - bind_user(connection, ldap, name, password) - after - :eldap.close(connection) + true -> + :ok end + bind_user(connection, ldap, name, password) + {:error, error} -> Logger.error("Could not open LDAP connection: #{inspect(error)}") {:error, {:ldap_connection_error, error}} From 91d1d7260b7084f59ae42e7c4b46c7fb963fda96 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 15 Sep 2024 23:18:17 -0400 Subject: [PATCH 029/100] Retain the try do so an LDAP failure can fall back to local database. This fixes tests but the automatic fallback may not be well documented behavior. --- lib/pleroma/web/auth/ldap_authenticator.ex | 42 ++++++++++++---------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 18a4e81ee4..ad5bc9863f 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -65,30 +65,34 @@ defp ldap_user(name, password) do case :eldap.open([to_charlist(host)], options) do {:ok, connection} -> - cond do - ssl -> - :application.ensure_all_started(:ssl) + try do + cond do + ssl -> + :application.ensure_all_started(:ssl) - tls -> - case :eldap.start_tls( - connection, - tlsopts, - @connection_timeout - ) do - :ok -> - :ok + tls -> + case :eldap.start_tls( + connection, + tlsopts, + @connection_timeout + ) do + :ok -> + :ok - error -> - Logger.error("Could not start TLS: #{inspect(error)}") - :eldap.close(connection) - end + error -> + Logger.error("Could not start TLS: #{inspect(error)}") + :eldap.close(connection) + end - true -> - :ok + true -> + :ok + end + + bind_user(connection, ldap, name, password) + after + :eldap.close(connection) end - bind_user(connection, ldap, name, password) - {:error, error} -> Logger.error("Could not open LDAP connection: #{inspect(error)}") {:error, {:ldap_connection_error, error}} From 0366bb284c32f6bec747845be50dcf8adc333712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 16 Sep 2024 11:42:09 +0200 Subject: [PATCH 030/100] Fix migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- ..._report_notification_type_to_admin_report.exs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs b/priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs index d06addf067..52cf3fac43 100644 --- a/priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs +++ b/priv/repo/migrations/20240912000000_rename_pleroma_report_notification_type_to_admin_report.exs @@ -36,10 +36,10 @@ def up do 'poll', 'status', 'update', - 'pleroma:participation_accepted' - 'pleroma:participation_request' - 'pleroma:event_reminder' - 'pleroma:event_update' + 'pleroma:participation_accepted', + 'pleroma:participation_request', + 'pleroma:event_reminder', + 'pleroma:event_update', 'bite' ) """ @@ -83,10 +83,10 @@ def down do 'poll', 'status', 'update', - 'pleroma:participation_accepted' - 'pleroma:participation_request' - 'pleroma:event_reminder' - 'pleroma:event_update' + 'pleroma:participation_accepted', + 'pleroma:participation_request', + 'pleroma:event_reminder', + 'pleroma:event_update', 'bite' ) """ From 7694ca4b259631567697c1b884f5ed084a9550e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 16 Sep 2024 11:56:02 +0200 Subject: [PATCH 031/100] Support Mozhi as translation provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/description.exs | 13 +++ lib/pleroma/language/translation/deepl.ex | 2 +- .../language/translation/libretranslate.ex | 2 +- lib/pleroma/language/translation/mozhi.ex | 109 ++++++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/language/translation/mozhi.ex diff --git a/config/description.exs b/config/description.exs index 50637af49f..c88d1c5161 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3577,6 +3577,19 @@ "translateLocally intermediate language (used when direct source->target model is not available)", type: :string, suggestions: ["en"] + }, + %{ + group: {:subgroup, Pleroma.Language.Translation.Mozhi}, + key: :base_url, + label: "Mozhi instance URL", + type: :string + }, + %{ + group: {:subgroup, Pleroma.Language.Translation.Mozhi}, + key: :engine, + label: "Engine used for Mozhi", + type: :string, + suggestions: ["libretranslate"] } ] }, diff --git a/lib/pleroma/language/translation/deepl.ex b/lib/pleroma/language/translation/deepl.ex index e027035b4d..98b2a7e369 100644 --- a/lib/pleroma/language/translation/deepl.ex +++ b/lib/pleroma/language/translation/deepl.ex @@ -30,7 +30,7 @@ def translate(content, source_language, target_language) do text: content, source_lang: source_language |> String.upcase(), target_lang: target_language, - tag_handling: "html" + tag_handling: @name }), "", [ diff --git a/lib/pleroma/language/translation/libretranslate.ex b/lib/pleroma/language/translation/libretranslate.ex index 69ecf23b0b..fd727d1cf2 100644 --- a/lib/pleroma/language/translation/libretranslate.ex +++ b/lib/pleroma/language/translation/libretranslate.ex @@ -46,7 +46,7 @@ def translate(content, source_language, target_language) do %{ content: content, detected_source_language: source_language, - provider: "LibreTranslate" + provider: @name }} _ -> diff --git a/lib/pleroma/language/translation/mozhi.ex b/lib/pleroma/language/translation/mozhi.ex new file mode 100644 index 0000000000..958f2ef57b --- /dev/null +++ b/lib/pleroma/language/translation/mozhi.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Language.Translation.Mozhi do + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + + alias Pleroma.Language.Translation.Provider + + use Provider + + @behaviour Provider + + @name "Mozhi" + + @impl Provider + def configured?, do: not_empty_string(base_url()) and not_empty_string(engine()) + + @impl Provider + def translate(content, source_language, target_language) do + endpoint = + base_url() + |> URI.merge("/api/translate") + |> URI.to_string() + + case Pleroma.HTTP.get( + endpoint <> + "?" <> + URI.encode_query(%{ + engine: engine(), + text: content, + from: source_language, + to: target_language + }), + [{"Accept", "application/json"}] + ) do + {:ok, %{status: 200} = res} -> + %{ + "translated-text" => content, + "source_language" => source_language + } = Jason.decode!(res.body) + + {:ok, + %{ + content: content, + detected_source_language: source_language, + provider: @name + }} + + _ -> + {:error, :internal_server_error} + end + end + + @impl Provider + def supported_languages(type) when type in [:source, :target] do + path = + case type do + :source -> "/api/source_languages" + :target -> "/api/target_languages" + end + + endpoint = + base_url() + |> URI.merge(path) + |> URI.to_string() + + case Pleroma.HTTP.get( + endpoint <> + "?" <> + URI.encode_query(%{ + engine: engine() + }), + [{"Accept", "application/json"}] + ) do + {:ok, %{status: 200} = res} -> + languages = + Jason.decode!(res.body) + |> Enum.map(fn %{"Id" => language} -> language end) + + {:ok, languages} + + _ -> + {:error, :internal_server_error} + end + end + + @impl Provider + def languages_matrix do + with {:ok, source_languages} <- supported_languages(:source), + {:ok, target_languages} <- supported_languages(:target) do + {:ok, + Map.new(source_languages, fn language -> {language, target_languages -- [language]} end)} + else + {:error, error} -> {:error, error} + end + end + + @impl Provider + def name, do: @name + + defp base_url do + Pleroma.Config.get([__MODULE__, :base_url]) + end + + defp engine do + Pleroma.Config.get([__MODULE__, :engine]) + end +end From e74e0089bf2943f925cbead14154f8b2fa207963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 16 Sep 2024 17:07:39 +0200 Subject: [PATCH 032/100] Repesct :restrict_unauthenticated for hashtag rss/atom feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/hashtag-feeds-restricted.add | 1 + lib/pleroma/web/feed/tag_controller.ex | 6 +- test/pleroma/web/feed/tag_controller_test.exs | 56 +++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 changelog.d/hashtag-feeds-restricted.add diff --git a/changelog.d/hashtag-feeds-restricted.add b/changelog.d/hashtag-feeds-restricted.add new file mode 100644 index 0000000000..accac9c9cc --- /dev/null +++ b/changelog.d/hashtag-feeds-restricted.add @@ -0,0 +1 @@ +Repesct :restrict_unauthenticated for hashtag rss/atom feeds \ No newline at end of file diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index e60767327c..02d6392960 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Feed.TagController do alias Pleroma.Web.Feed.FeedView def feed(conn, params) do - if Config.get!([:instance, :public]) do + if not Config.restrict_unauthenticated_access?(:timelines, :local) do render_feed(conn, params) else render_error(conn, :not_found, "Not found") @@ -18,10 +18,12 @@ def feed(conn, params) do end defp render_feed(conn, %{"tag" => raw_tag} = params) do + local_only = Config.restrict_unauthenticated_access?(:timelines, :federated) + {format, tag} = parse_tag(raw_tag) activities = - %{type: ["Create"], tag: tag} + %{type: ["Create"], tag: tag, local_only: local_only} |> Pleroma.Maps.put_if_present(:max_id, params["max_id"]) |> ActivityPub.fetch_public_activities() diff --git a/test/pleroma/web/feed/tag_controller_test.exs b/test/pleroma/web/feed/tag_controller_test.exs index 7d196b228e..662235f31f 100644 --- a/test/pleroma/web/feed/tag_controller_test.exs +++ b/test/pleroma/web/feed/tag_controller_test.exs @@ -191,4 +191,60 @@ test "returns 404 for tags feed", %{conn: conn} do |> response(404) end end + + describe "restricted for unauthenticated" do + test "returns 404 when local timeline is disabled", %{conn: conn} do + clear_config([:restrict_unauthenticated, :timelines], %{local: true, federated: false}) + + conn + |> put_req_header("accept", "application/rss+xml") + |> get(tag_feed_path(conn, :feed, "pleromaart.rss")) + |> response(404) + end + + test "returns local posts only when federated timeline is disabled", %{conn: conn} do + clear_config([:restrict_unauthenticated, :timelines], %{local: false, federated: true}) + + local_user = insert(:user) + remote_user = insert(:user, local: false) + + local_note = + insert(:note, + user: local_user, + data: %{ + "content" => "local post #PleromaArt", + "summary" => "", + "tag" => ["pleromaart"] + } + ) + + remote_note = + insert(:note, + user: remote_user, + data: %{ + "content" => "remote post #PleromaArt", + "summary" => "", + "tag" => ["pleromaart"] + }, + local: false + ) + + insert(:note_activity, user: local_user, note: local_note) + insert(:note_activity, user: remote_user, note: remote_note, local: false) + + response = + conn + |> put_req_header("accept", "application/rss+xml") + |> get(tag_feed_path(conn, :feed, "pleromaart.rss")) + |> response(200) + + xml = parse(response) + + assert xpath(xml, ~x"//channel/title/text()") == ~c"#pleromaart" + + assert xpath(xml, ~x"//channel/item/title/text()"l) == [ + ~c"local post #PleromaArt" + ] + end + end end From e59706c201bd71525c0a15008c3cb5dcdfb73289 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 11:39:19 -0400 Subject: [PATCH 033/100] Reapply "Custom mix task to retry failed tests once in CI pipeline" This reverts commit b281ad06de2de331450a5e319e3ba497071d4197. --- .gitlab-ci.yml | 2 +- changelog.d/fix-test-failures.skip | 0 lib/mix/tasks/pleroma/test_runner.ex | 25 +++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fix-test-failures.skip create mode 100644 lib/mix/tasks/pleroma/test_runner.ex diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e04dae76f..76d1a4210d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,7 +134,7 @@ unit-testing-1.13.4-otp-25: script: &testing_script - mix ecto.create - mix ecto.migrate - - mix test --cover --preload-modules + - mix pleroma.test_runner --cover --preload-modules coverage: '/^Line total: ([^ ]*%)$/' artifacts: reports: diff --git a/changelog.d/fix-test-failures.skip b/changelog.d/fix-test-failures.skip deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex new file mode 100644 index 0000000000..69fefb0014 --- /dev/null +++ b/lib/mix/tasks/pleroma/test_runner.ex @@ -0,0 +1,25 @@ +defmodule Mix.Tasks.Pleroma.TestRunner do + @shortdoc "Retries tests once if they fail" + + use Mix.Task + + def run(args \\ []) do + case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + retry(args) + end + end + + def retry(args) do + case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + exit(1) + end + end +end From f3f01f4093dfce773663eb44b36fa787f265527c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 17 Sep 2024 13:48:45 +0200 Subject: [PATCH 034/100] Change color for avatar border MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- priv/static/instance/static.css | Bin 6741 -> 6747 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/priv/static/instance/static.css b/priv/static/instance/static.css index a162e0f2ade762299be8bc2e7fbb39a0e98c482e..183f9156dda27f3630ea50deea80e956aa5e1e4b 100644 GIT binary patch delta 16 Xcmca=a@%A>tng$bNyW_-!gtvKJC6og delta 16 Xcmca@a@Ax*tng$94(ZKx!gtvKJBS8W From 9264b21907f5c6890694d6d611ade9b13433463a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 00:26:57 -0400 Subject: [PATCH 035/100] Pleroma.LDAP This adds a GenServer which will keep an LDAP connection open and auto reconnect on failure with a 5 second wait between retries. Another benefit is this prevents parsing the Root CAs for every login attempt as we only need to do it once per connection. --- lib/pleroma/application.ex | 1 + lib/pleroma/ldap.ex | 233 +++++++++++++++++++++ lib/pleroma/web/auth/ldap_authenticator.ex | 147 +------------ 3 files changed, 236 insertions(+), 145 deletions(-) create mode 100644 lib/pleroma/ldap.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index cb15dc1e9a..3f199c002d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -94,6 +94,7 @@ def start(_type, _args) do children = [ Pleroma.PromEx, + Pleroma.LDAP, Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex new file mode 100644 index 0000000000..8689325012 --- /dev/null +++ b/lib/pleroma/ldap.ex @@ -0,0 +1,233 @@ +defmodule Pleroma.LDAP do + use GenServer + + require Logger + + alias Pleroma.Config + alias Pleroma.User + + import Pleroma.Web.Auth.Helpers, only: [fetch_user: 1] + + @connection_timeout 10_000 + @search_timeout 10_000 + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(state) do + case {Config.get(Pleroma.Web.Auth.Authenticator), Config.get([:ldap, :enabled])} do + {Pleroma.Web.Auth.LDAPAuthenticator, true} -> + {:ok, state, {:continue, :connect}} + + {Pleroma.Web.Auth.LDAPAuthenticator, false} -> + Logger.error( + "LDAP Authenticator enabled but :pleroma, :ldap is not enabled. Auth will not work." + ) + + {:ok, state} + + {_, true} -> + Logger.warning( + ":pleroma, :ldap is enabled but Pleroma.Web.Authenticator is not set to the LDAPAuthenticator. LDAP will not be used." + ) + + {:ok, state} + end + end + + @impl true + def handle_continue(:connect, _state), do: do_handle_connect() + + @impl true + def handle_info(:connect, _state), do: do_handle_connect() + + def handle_info({:bind_after_reconnect, name, password, from}, state) do + result = bind_user(state[:connection], name, password) + + GenServer.reply(from, result) + + {:noreply, state} + end + + defp do_handle_connect() do + state = + case connect() do + {:ok, connection} -> + :eldap.controlling_process(connection, self()) + [connection: connection] + + _ -> + Logger.error("Failed to connect to LDAP. Retrying in 5000ms") + Process.send_after(self(), :connect, 5_000) + [] + end + + {:noreply, state} + end + + @impl true + def handle_call({:bind_user, name, password}, from, state) do + case bind_user(state[:connection], name, password) do + :needs_reconnect -> + Process.send(self(), {:bind_after_reconnect, name, password, from}, []) + {:noreply, state, {:continue, :connect}} + + result -> + {:reply, result, state, :hibernate} + end + end + + @impl true + def terminate(_, state) do + :eldap.close(state[:connection]) + + :ok + end + + defp connect() do + ldap = Config.get(:ldap, []) + host = Keyword.get(ldap, :host, "localhost") + port = Keyword.get(ldap, :port, 389) + ssl = Keyword.get(ldap, :ssl, false) + tls = Keyword.get(ldap, :tls, false) + cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() + + default_secure_opts = [ + verify: :verify_peer, + cacerts: decode_certfile(cacertfile), + customize_hostname_check: [ + fqdn_fun: fn _ -> to_charlist(host) end + ] + ] + + sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, [])) + tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, [])) + + default_options = [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] + + # :sslopts can only be included in :eldap.open/2 when {ssl: true} + # or the connection will fail + options = + if ssl do + default_options ++ [{:sslopts, sslopts}] + else + default_options + end + + case :eldap.open([to_charlist(host)], options) do + {:ok, connection} -> + try do + cond do + ssl -> + :application.ensure_all_started(:ssl) + {:ok, connection} + + tls -> + case :eldap.start_tls( + connection, + tlsopts, + @connection_timeout + ) do + :ok -> + {:ok, connection} + + error -> + Logger.error("Could not start TLS: #{inspect(error)}") + :eldap.close(connection) + end + + true -> + {:ok, :connection} + end + after + :ok + end + + {:error, error} -> + Logger.error("Could not open LDAP connection: #{inspect(error)}") + {:error, {:ldap_connection_error, error}} + end + end + + defp bind_user(connection, name, password) do + uid = Config.get([:ldap, :uid], "cn") + base = Config.get([:ldap, :base]) + + case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do + :ok -> + case fetch_user(name) do + %User{} = user -> + user + + _ -> + register_user(connection, base, uid, name) + end + + # eldap does not inform us of socket closure + # until it is used + {:error, {:gen_tcp_error, :closed}} -> + :eldap.close(connection) + :needs_reconnect + + error -> + Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}") + {:error, {:ldap_bind_error, error}} + end + end + + defp register_user(connection, base, uid, name) do + case :eldap.search(connection, [ + {:base, to_charlist(base)}, + {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))}, + {:scope, :eldap.wholeSubtree()}, + {:timeout, @search_timeout} + ]) do + # The :eldap_search_result record structure changed in OTP 24.3 and added a controls field + # https://github.com/erlang/otp/pull/5538 + {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals}} -> + try_register(name, attributes) + + {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals, _controls}} -> + try_register(name, attributes) + + error -> + Logger.error("Couldn't register user because LDAP search failed: #{inspect(error)}") + {:error, {:ldap_search_error, error}} + end + end + + defp try_register(name, attributes) do + params = %{ + name: name, + nickname: name, + password: nil + } + + params = + case List.keyfind(attributes, ~c"mail", 0) do + {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) + _ -> params + end + + changeset = User.register_changeset_ldap(%User{}, params) + + case User.register(changeset) do + {:ok, user} -> user + error -> error + end + end + + defp decode_certfile(file) do + with {:ok, data} <- File.read(file) do + data + |> :public_key.pem_decode() + |> Enum.map(fn {_, b, _} -> b end) + else + _ -> + Logger.error("Unable to read certfile: #{file}") + [] + end + end +end diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index ad5bc9863f..c420c8bc30 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -5,16 +5,11 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do alias Pleroma.User - require Logger - - import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1, fetch_user: 1] + import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1] @behaviour Pleroma.Web.Auth.Authenticator @base Pleroma.Web.Auth.PleromaAuthenticator - @connection_timeout 10_000 - @search_timeout 10_000 - defdelegate get_registration(conn), to: @base defdelegate create_from_registration(conn, registration), to: @base defdelegate handle_error(conn, error), to: @base @@ -24,7 +19,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do def get_user(%Plug.Conn{} = conn) do with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])}, {:ok, {name, password}} <- fetch_credentials(conn), - %User{} = user <- ldap_user(name, password) do + %User{} = user <- GenServer.call(Pleroma.LDAP, {:bind_user, name, password}) do {:ok, user} else {:ldap, _} -> @@ -34,142 +29,4 @@ def get_user(%Plug.Conn{} = conn) do error end end - - defp ldap_user(name, password) do - ldap = Pleroma.Config.get(:ldap, []) - host = Keyword.get(ldap, :host, "localhost") - port = Keyword.get(ldap, :port, 389) - ssl = Keyword.get(ldap, :ssl, false) - tls = Keyword.get(ldap, :tls, false) - cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() - - default_secure_opts = [ - verify: :verify_peer, - cacerts: decode_certfile(cacertfile), - customize_hostname_check: [ - fqdn_fun: fn _ -> to_charlist(host) end - ] - ] - - sslopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :sslopts, [])) - tlsopts = Keyword.merge(default_secure_opts, Keyword.get(ldap, :tlsopts, [])) - - # :sslopts can only be included in :eldap.open/2 when {ssl: true} - # or the connection will fail - options = - if ssl do - [{:port, port}, {:ssl, ssl}, {:sslopts, sslopts}, {:timeout, @connection_timeout}] - else - [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] - end - - case :eldap.open([to_charlist(host)], options) do - {:ok, connection} -> - try do - cond do - ssl -> - :application.ensure_all_started(:ssl) - - tls -> - case :eldap.start_tls( - connection, - tlsopts, - @connection_timeout - ) do - :ok -> - :ok - - error -> - Logger.error("Could not start TLS: #{inspect(error)}") - :eldap.close(connection) - end - - true -> - :ok - end - - bind_user(connection, ldap, name, password) - after - :eldap.close(connection) - end - - {:error, error} -> - Logger.error("Could not open LDAP connection: #{inspect(error)}") - {:error, {:ldap_connection_error, error}} - end - end - - defp bind_user(connection, ldap, name, password) do - uid = Keyword.get(ldap, :uid, "cn") - base = Keyword.get(ldap, :base) - - case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do - :ok -> - case fetch_user(name) do - %User{} = user -> - user - - _ -> - register_user(connection, base, uid, name) - end - - error -> - Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}") - {:error, {:ldap_bind_error, error}} - end - end - - defp register_user(connection, base, uid, name) do - case :eldap.search(connection, [ - {:base, to_charlist(base)}, - {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))}, - {:scope, :eldap.wholeSubtree()}, - {:timeout, @search_timeout} - ]) do - # The :eldap_search_result record structure changed in OTP 24.3 and added a controls field - # https://github.com/erlang/otp/pull/5538 - {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals}} -> - try_register(name, attributes) - - {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals, _controls}} -> - try_register(name, attributes) - - error -> - Logger.error("Couldn't register user because LDAP search failed: #{inspect(error)}") - {:error, {:ldap_search_error, error}} - end - end - - defp try_register(name, attributes) do - params = %{ - name: name, - nickname: name, - password: nil - } - - params = - case List.keyfind(attributes, ~c"mail", 0) do - {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) - _ -> params - end - - changeset = User.register_changeset_ldap(%User{}, params) - - case User.register(changeset) do - {:ok, user} -> user - error -> error - end - end - - defp decode_certfile(file) do - with {:ok, data} <- File.read(file) do - data - |> :public_key.pem_decode() - |> Enum.map(fn {_, b, _} -> b end) - else - _ -> - Logger.error("Unable to read certfile: #{file}") - [] - end - end end From ead287d623e83b8d9ffaa327b9edf96e046bfacd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 13:14:19 -0400 Subject: [PATCH 036/100] Credo --- lib/pleroma/ldap.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 8689325012..11544f0d9e 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -51,7 +51,7 @@ def handle_info({:bind_after_reconnect, name, password, from}, state) do {:noreply, state} end - defp do_handle_connect() do + defp do_handle_connect do state = case connect() do {:ok, connection} -> @@ -86,7 +86,7 @@ def terminate(_, state) do :ok end - defp connect() do + defp connect do ldap = Config.get(:ldap, []) host = Keyword.get(ldap, :host, "localhost") port = Keyword.get(ldap, :port, 389) From 7c04098dde0681f7ad299782bc09eaa9bc3a6bad Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:15:53 -0400 Subject: [PATCH 037/100] Catchall for when LDAP is not enabled --- lib/pleroma/ldap.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 11544f0d9e..ac819613e0 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -34,6 +34,9 @@ def init(state) do ) {:ok, state} + + _ -> + {:ok, state} end end From 44b836c94c1059551fbc7564770001311d2d1e6a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:24:27 -0400 Subject: [PATCH 038/100] Fix tests We do not need to mock and verify connections are closed as the new Pleroma.LDAP GenServer will handle managing the connection lifetime --- .../web/o_auth/ldap_authorization_test.exs | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/test/pleroma/web/o_auth/ldap_authorization_test.exs b/test/pleroma/web/o_auth/ldap_authorization_test.exs index 07ce2eed84..35b947fd04 100644 --- a/test/pleroma/web/o_auth/ldap_authorization_test.exs +++ b/test/pleroma/web/o_auth/ldap_authorization_test.exs @@ -28,11 +28,7 @@ test "authorizes the existing user using LDAP credentials" do {:eldap, [], [ open: fn [^host], [{:port, ^port}, {:ssl, false} | _] -> {:ok, self()} end, - simple_bind: fn _connection, _dn, ^password -> :ok end, - close: fn _connection -> - send(self(), :close_connection) - :ok - end + simple_bind: fn _connection, _dn, ^password -> :ok end ]} ] do conn = @@ -50,7 +46,6 @@ test "authorizes the existing user using LDAP credentials" do token = Repo.get_by(Token, token: token) assert token.user_id == user.id - assert_received :close_connection end end @@ -72,10 +67,6 @@ test "creates a new user after successful LDAP authorization" do wholeSubtree: fn -> :ok end, search: fn _connection, _options -> {:ok, {:eldap_search_result, [{:eldap_entry, ~c"", []}], []}} - end, - close: fn _connection -> - send(self(), :close_connection) - :ok end ]} ] do @@ -94,7 +85,6 @@ test "creates a new user after successful LDAP authorization" do token = Repo.get_by(Token, token: token) |> Repo.preload(:user) assert token.user.nickname == user.nickname - assert_received :close_connection end end @@ -111,11 +101,7 @@ test "disallow authorization for wrong LDAP credentials" do {:eldap, [], [ open: fn [^host], [{:port, ^port}, {:ssl, false} | _] -> {:ok, self()} end, - simple_bind: fn _connection, _dn, ^password -> {:error, :invalidCredentials} end, - close: fn _connection -> - send(self(), :close_connection) - :ok - end + simple_bind: fn _connection, _dn, ^password -> {:error, :invalidCredentials} end ]} ] do conn = @@ -129,7 +115,6 @@ test "disallow authorization for wrong LDAP credentials" do }) assert %{"error" => "Invalid credentials"} = json_response(conn, 400) - assert_received :close_connection end end end From d82abf925ddbe8b98ba8191713115db50c38a0c0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:25:44 -0400 Subject: [PATCH 039/100] Ensure :cacertfile is configurable in ConfigDB --- config/description.exs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/description.exs b/config/description.exs index 15faecb38c..ade47b7e02 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2297,6 +2297,12 @@ description: "LDAP attribute name to authenticate the user, e.g. when \"cn\", the filter will be \"cn=username,base\"", suggestions: ["cn"] + }, + %{ + key: :cacertfile, + label: "CACertfile", + type: :string, + description: "Path to CA certificate file" } ] }, From 65a7b387c35b4913b6109692a84bae80af8b9a96 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Sep 2024 16:28:37 -0400 Subject: [PATCH 040/100] Require a reboot if LDAP configuration changes --- lib/pleroma/config/transfer_task.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index ffc95f1447..140dd77111 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -22,7 +22,8 @@ defp reboot_time_keys, {:pleroma, :markup}, {:pleroma, :streamer}, {:pleroma, :pools}, - {:pleroma, :connections_pool} + {:pleroma, :connections_pool}, + {:pleroma, :ldap} ] defp reboot_time_subkeys, From 123093a1868b25f101dfc4b02895c22a0daf5733 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:07:26 -0400 Subject: [PATCH 041/100] Ensure :ssl is started before we attempt to make the LDAP connection --- lib/pleroma/ldap.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index ac819613e0..042a4daa21 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -97,6 +97,8 @@ defp connect do tls = Keyword.get(ldap, :tls, false) cacertfile = Keyword.get(ldap, :cacertfile) || CAStore.file_path() + if ssl, do: Application.ensure_all_started(:ssl) + default_secure_opts = [ verify: :verify_peer, cacerts: decode_certfile(cacertfile), @@ -123,10 +125,6 @@ defp connect do {:ok, connection} -> try do cond do - ssl -> - :application.ensure_all_started(:ssl) - {:ok, connection} - tls -> case :eldap.start_tls( connection, From d0ee899ab94788e37e6ac3c43342017d1b27903a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:14:09 -0400 Subject: [PATCH 042/100] Only close connection if it is not nil --- lib/pleroma/ldap.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 042a4daa21..3df20fe094 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -84,7 +84,11 @@ def handle_call({:bind_user, name, password}, from, state) do @impl true def terminate(_, state) do - :eldap.close(state[:connection]) + connection = Keyword.get(state, :connection) + + if not is_nil(connection) do + :eldap.close(connection) + end :ok end From 164ffbcab822eda4c28f912082b6a7a3ec64a7e5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:17:40 -0400 Subject: [PATCH 043/100] Fix return value when not doing STARTTLS --- lib/pleroma/ldap.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 3df20fe094..6be2188f43 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -144,7 +144,7 @@ defp connect do end true -> - {:ok, :connection} + {:ok, connection} end after :ok From a1972d57e30f41e5173d61c0d0936685738c560d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:19:54 -0400 Subject: [PATCH 044/100] Link the eldap connection process Ensure if LDAP GenServer crashes it gets cleaned up, and we should crash and restart if somehow the eldap connection process crashes unexpectedly as we can't seem to receive any DOWN messages from it, etc. --- lib/pleroma/ldap.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 6be2188f43..0723cd0941 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -59,6 +59,7 @@ defp do_handle_connect do case connect() do {:ok, connection} -> :eldap.controlling_process(connection, self()) + Process.link(connection) [connection: connection] _ -> From 14a9663f1abe49b8f4f4f719fa2f4db3a5dd81b7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:28:42 -0400 Subject: [PATCH 045/100] Remove cacertfile as child of SSL and TLS options We need to pass the cacerts (list of charlist encoded certs) not cacertfile, so our new cacertfile setting handles this for us. --- config/description.exs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/config/description.exs b/config/description.exs index ade47b7e02..5062842f04 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2241,14 +2241,8 @@ label: "SSL options", type: :keyword, description: "Additional SSL options", - suggestions: [cacertfile: "path/to/file/with/PEM/cacerts", verify: :verify_peer], + suggestions: [verify: :verify_peer], children: [ - %{ - key: :cacertfile, - type: :string, - description: "Path to file with PEM encoded cacerts", - suggestions: ["path/to/file/with/PEM/cacerts"] - }, %{ key: :verify, type: :atom, @@ -2268,14 +2262,8 @@ label: "TLS options", type: :keyword, description: "Additional TLS options", - suggestions: [cacertfile: "path/to/file/with/PEM/cacerts", verify: :verify_peer], + suggestions: [verify: :verify_peer], children: [ - %{ - key: :cacertfile, - type: :string, - description: "Path to file with PEM encoded cacerts", - suggestions: ["path/to/file/with/PEM/cacerts"] - }, %{ key: :verify, type: :atom, From 363b462c54c454e847072869db09f8f4d5da4426 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:36:46 -0400 Subject: [PATCH 046/100] Make the email attribute configurable While here, fix the System.get_env usage to use the normal fallback value method and improve the UID label description --- config/config.exs | 11 ++++++----- config/description.exs | 9 ++++++++- lib/pleroma/ldap.ex | 4 +++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index f53a083d04..47ddfac5ac 100644 --- a/config/config.exs +++ b/config/config.exs @@ -612,16 +612,17 @@ config :pleroma, :ldap, enabled: System.get_env("LDAP_ENABLED") == "true", - host: System.get_env("LDAP_HOST") || "localhost", - port: String.to_integer(System.get_env("LDAP_PORT") || "389"), + host: System.get_env("LDAP_HOST", "localhost"), + port: String.to_integer(System.get_env("LDAP_PORT", "389")), ssl: System.get_env("LDAP_SSL") == "true", sslopts: [], tls: System.get_env("LDAP_TLS") == "true", tlsopts: [], - base: System.get_env("LDAP_BASE") || "dc=example,dc=com", - uid: System.get_env("LDAP_UID") || "cn", + base: System.get_env("LDAP_BASE", "dc=example,dc=com"), + uid: System.get_env("LDAP_UID", "cn"), # defaults to CAStore's Mozilla roots - cacertfile: nil + cacertfile: System.get_env("LDAP_CACERTFILE", nil), + mail: System.get_env("LDAP_MAIL", "mail") oauth_consumer_strategies = System.get_env("OAUTH_CONSUMER_STRATEGIES") diff --git a/config/description.exs b/config/description.exs index 5062842f04..e85ec0ff80 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2280,7 +2280,7 @@ }, %{ key: :uid, - label: "UID", + label: "UID Attribute", type: :string, description: "LDAP attribute name to authenticate the user, e.g. when \"cn\", the filter will be \"cn=username,base\"", @@ -2291,6 +2291,13 @@ label: "CACertfile", type: :string, description: "Path to CA certificate file" + }, + %{ + key: :mail, + label: "Mail Attribute", + type: :string, + description: "LDAP attribute name to use as the email address when automatically registering the user on first login", + suggestions: ["mail"] } ] }, diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 0723cd0941..8e9c591b29 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -205,6 +205,8 @@ defp register_user(connection, base, uid, name) do end defp try_register(name, attributes) do + mail_attribute = Config.get([:ldap, :mail]) + params = %{ name: name, nickname: name, @@ -212,7 +214,7 @@ defp try_register(name, attributes) do } params = - case List.keyfind(attributes, ~c"mail", 0) do + case List.keyfind(attributes, to_charlist(mail_attribute), 0) do {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) _ -> params end From 21bf229731f27426564140650397e51ba4bb4b93 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:43:21 -0400 Subject: [PATCH 047/100] Reduce LDAP timeouts 10 seconds is way too long for any login attempt or search result. LDAP should always be fast. --- lib/pleroma/ldap.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 8e9c591b29..33c9cff295 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -8,8 +8,8 @@ defmodule Pleroma.LDAP do import Pleroma.Web.Auth.Helpers, only: [fetch_user: 1] - @connection_timeout 10_000 - @search_timeout 10_000 + @connection_timeout 2_000 + @search_timeout 2_000 def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) From 1d123832da6a2b8c67f34006b4ea05e0be86e366 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:46:49 -0400 Subject: [PATCH 048/100] Formatting --- config/description.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index e85ec0ff80..47f4771eb8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2296,7 +2296,8 @@ key: :mail, label: "Mail Attribute", type: :string, - description: "LDAP attribute name to use as the email address when automatically registering the user on first login", + description: + "LDAP attribute name to use as the email address when automatically registering the user on first login", suggestions: ["mail"] } ] From ea63533cf28be8218a27806b1f6430f0c1ca4a01 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:46:56 -0400 Subject: [PATCH 049/100] Change :connection to :handle to match upstream nomenclature --- lib/pleroma/ldap.ex | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 33c9cff295..b357b371ad 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -47,7 +47,7 @@ def handle_continue(:connect, _state), do: do_handle_connect() def handle_info(:connect, _state), do: do_handle_connect() def handle_info({:bind_after_reconnect, name, password, from}, state) do - result = bind_user(state[:connection], name, password) + result = bind_user(state[:handle], name, password) GenServer.reply(from, result) @@ -57,10 +57,10 @@ def handle_info({:bind_after_reconnect, name, password, from}, state) do defp do_handle_connect do state = case connect() do - {:ok, connection} -> - :eldap.controlling_process(connection, self()) - Process.link(connection) - [connection: connection] + {:ok, handle} -> + :eldap.controlling_process(handle, self()) + Process.link(handle) + [handle: handle] _ -> Logger.error("Failed to connect to LDAP. Retrying in 5000ms") @@ -73,7 +73,7 @@ defp do_handle_connect do @impl true def handle_call({:bind_user, name, password}, from, state) do - case bind_user(state[:connection], name, password) do + case bind_user(state[:handle], name, password) do :needs_reconnect -> Process.send(self(), {:bind_after_reconnect, name, password, from}, []) {:noreply, state, {:continue, :connect}} @@ -85,10 +85,10 @@ def handle_call({:bind_user, name, password}, from, state) do @impl true def terminate(_, state) do - connection = Keyword.get(state, :connection) + handle = Keyword.get(state, :handle) - if not is_nil(connection) do - :eldap.close(connection) + if not is_nil(handle) do + :eldap.close(handle) end :ok @@ -127,25 +127,25 @@ defp connect do end case :eldap.open([to_charlist(host)], options) do - {:ok, connection} -> + {:ok, handle} -> try do cond do tls -> case :eldap.start_tls( - connection, + handle, tlsopts, @connection_timeout ) do :ok -> - {:ok, connection} + {:ok, handle} error -> Logger.error("Could not start TLS: #{inspect(error)}") - :eldap.close(connection) + :eldap.close(handle) end true -> - {:ok, connection} + {:ok, handle} end after :ok @@ -157,24 +157,24 @@ defp connect do end end - defp bind_user(connection, name, password) do + defp bind_user(handle, name, password) do uid = Config.get([:ldap, :uid], "cn") base = Config.get([:ldap, :base]) - case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do + case :eldap.simple_bind(handle, "#{uid}=#{name},#{base}", password) do :ok -> case fetch_user(name) do %User{} = user -> user _ -> - register_user(connection, base, uid, name) + register_user(handle, base, uid, name) end # eldap does not inform us of socket closure # until it is used {:error, {:gen_tcp_error, :closed}} -> - :eldap.close(connection) + :eldap.close(handle) :needs_reconnect error -> @@ -183,8 +183,8 @@ defp bind_user(connection, name, password) do end end - defp register_user(connection, base, uid, name) do - case :eldap.search(connection, [ + defp register_user(handle, base, uid, name) do + case :eldap.search(handle, [ {:base, to_charlist(base)}, {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))}, {:scope, :eldap.wholeSubtree()}, From 2b482e34ebf5aee49350d8198e6fade8820b3834 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:54:57 -0400 Subject: [PATCH 050/100] Improve matching on bind errors --- lib/pleroma/ldap.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index b357b371ad..cd84dee023 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -177,9 +177,9 @@ defp bind_user(handle, name, password) do :eldap.close(handle) :needs_reconnect - error -> + {:error, error} = e -> Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}") - {:error, {:ldap_bind_error, error}} + e end end From 35ddb1d2c8a53dcb54178522811242ef40a63211 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 17 Sep 2024 13:57:10 -0400 Subject: [PATCH 051/100] LDAP genserver changelog --- changelog.d/ldap-refactor.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/ldap-refactor.change diff --git a/changelog.d/ldap-refactor.change b/changelog.d/ldap-refactor.change new file mode 100644 index 0000000000..1510eea6aa --- /dev/null +++ b/changelog.d/ldap-refactor.change @@ -0,0 +1 @@ +LDAP authentication has been refactored to operate as a GenServer process which will maintain an active connection to the LDAP server. From 1de5208a9e90485be38ac0d00088f18c5b36390a Mon Sep 17 00:00:00 2001 From: Mint Date: Tue, 17 Sep 2024 21:48:37 +0300 Subject: [PATCH 052/100] Cheatsheet: add Mua mail adapter config --- docs/configuration/cheatsheet.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 0b4e53b6f5..ba41fe84df 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -742,6 +742,21 @@ config :pleroma, Pleroma.Emails.Mailer, auth: :always ``` +An example for Mua adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + enabled: true, + adapter: Swoosh.Adapters.Mua, + relay: "mail.example.com", + port: 465, + auth: [ + username: "YOUR_USERNAME@domain.tld", + password: "YOUR_SMTP_PASSWORD" + ], + protocol: :ssl +``` + ### :email_notifications Email notifications settings. From 73204c1bca740dbca5c780891fc720ac728c11a6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 11:16:16 -0400 Subject: [PATCH 053/100] LDAP: fix compile warning Sometimes the compile will emit the following warning, so we'll just avoid it by making it call a function in the LDAP module which will never have this problem. warning: :GenServer.call/2 is undefined (module :GenServer is not available or is yet to be defined) --- changelog.d/ldap-warning.skip | 0 lib/pleroma/ldap.ex | 4 ++++ lib/pleroma/web/auth/ldap_authenticator.ex | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ldap-warning.skip diff --git a/changelog.d/ldap-warning.skip b/changelog.d/ldap-warning.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index cd84dee023..46a2d0c17b 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -94,6 +94,10 @@ def terminate(_, state) do :ok end + def bind_user(name, password) do + GenServer.call(__MODULE__, {:bind_user, name, password}) + end + defp connect do ldap = Config.get(:ldap, []) host = Keyword.get(ldap, :host, "localhost") diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index c420c8bc30..7eb06183d3 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Auth.LDAPAuthenticator do + alias Pleroma.LDAP alias Pleroma.User import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1] @@ -19,7 +20,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do def get_user(%Plug.Conn{} = conn) do with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])}, {:ok, {name, password}} <- fetch_credentials(conn), - %User{} = user <- GenServer.call(Pleroma.LDAP, {:bind_user, name, password}) do + %User{} = user <- LDAP.bind_user(name, password) do {:ok, user} else {:ldap, _} -> From ecd1b8393befe91175872af3db67a5c01f10eaf2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 12:07:52 -0400 Subject: [PATCH 054/100] Oban: update to 2.18.3 This release includes the fix which should prevent the scenario where Postgrex crashes can cause Oban to get into a state where it will stop processing jobs. --- changelog.d/oban-update.change | 1 + mix.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/oban-update.change diff --git a/changelog.d/oban-update.change b/changelog.d/oban-update.change new file mode 100644 index 0000000000..48a54ed2d2 --- /dev/null +++ b/changelog.d/oban-update.change @@ -0,0 +1 @@ +Oban updated to 2.18.3 diff --git a/mix.lock b/mix.lock index 2cf44862b4..421f99ec04 100644 --- a/mix.lock +++ b/mix.lock @@ -92,7 +92,7 @@ "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "2.18.2", "583e78965ee15263ac968e38c983bad169ae55eadaa8e1e39912562badff93ba", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9dd25fd35883a91ed995e9fe516e479344d3a8623dfe2b8c3fc8e5be0228ec3a"}, + "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"}, "oban_live_dashboard": {:hex, :oban_live_dashboard, "0.1.1", "8aa4ceaf381c818f7d5c8185cc59942b8ac82ef0cf559881aacf8d3f8ac7bdd3", [:mix], [{:oban, "~> 2.15", [hex: :oban, repo: "hexpm", optional: false]}, {:phoenix_live_dashboard, "~> 0.7", [hex: :phoenix_live_dashboard, repo: "hexpm", optional: false]}], "hexpm", "16dc4ce9c9a95aa2e655e35ed4e675652994a8def61731a18af85e230e1caa63"}, "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, "open_api_spex": {:hex, :open_api_spex, "3.18.2", "8c855e83bfe8bf81603d919d6e892541eafece3720f34d1700b58024dadde247", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "aa3e6dcfc0ad6a02596b2172662da21c9dd848dac145ea9e603f54e3d81b8d2b"}, From f00545d85bd601734cdbbc28454f33541dbf530d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 13:14:17 -0400 Subject: [PATCH 055/100] Elixir 1.14 and Erlang/OTP 23 is now the minimum supported release --- .gitlab-ci.yml | 8 ++++---- changelog.d/elixir.change | 1 + docs/installation/debian_based_jp.md | 2 +- docs/installation/generic_dependencies.include | 4 ++-- mix.exs | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changelog.d/elixir.change diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76d1a4210d..39947c75e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,8 @@ -image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25 +image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.14.5-otp-25 variables: &global_variables # Only used for the release - ELIXIR_VER: 1.13.4 + ELIXIR_VER: 1.14.5 POSTGRES_DB: pleroma_test POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -71,7 +71,7 @@ check-changelog: tags: - amd64 -build-1.13.4-otp-25: +build-1.14.5-otp-25: extends: - .build_changes_policy - .using-ci-base @@ -119,7 +119,7 @@ benchmark: - mix ecto.migrate - mix pleroma.load_testing -unit-testing-1.13.4-otp-25: +unit-testing-1.14.5-otp-25: extends: - .build_changes_policy - .using-ci-base diff --git a/changelog.d/elixir.change b/changelog.d/elixir.change new file mode 100644 index 0000000000..779c01562b --- /dev/null +++ b/changelog.d/elixir.change @@ -0,0 +1 @@ +Elixir 1.14 and Erlang/OTP 23 is now the minimum supported release diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 5a0823a634..0817934fff 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have - PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) - `postgresql-contrib` 11.0以上 (同上) -- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) +- Elixir 1.14 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` - `git` diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index bdb7f94d3a..9f07f62c6c 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,8 +1,8 @@ ## Required dependencies * PostgreSQL >=11.0 -* Elixir >=1.13.0 <1.17 -* Erlang OTP >=22.2.0 (supported: <27) +* Elixir >=1.14.0 <1.17 +* Erlang OTP >=23.0.0 (supported: <27) * git * file / libmagic * gcc or clang diff --git a/mix.exs b/mix.exs index ceae5c26df..89ec5e831e 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ def project do [ app: :pleroma, version: version("2.7.0"), - elixir: "~> 1.13", + elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors(), prune_code_paths: false], From 7e303600fb2914ab66fe54a8022ddc65ff93edf5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 17:15:55 +0000 Subject: [PATCH 056/100] Remove old elixir 1.12 build image generation script --- ci/elixir-1.12/Dockerfile | 8 -------- ci/elixir-1.12/build_and_push.sh | 1 - 2 files changed, 9 deletions(-) delete mode 100644 ci/elixir-1.12/Dockerfile delete mode 100755 ci/elixir-1.12/build_and_push.sh diff --git a/ci/elixir-1.12/Dockerfile b/ci/elixir-1.12/Dockerfile deleted file mode 100644 index a2b5668730..0000000000 --- a/ci/elixir-1.12/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM elixir:1.12.3 - -# Single RUN statement, otherwise intermediate images are created -# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run -RUN apt-get update &&\ - apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\ - mix local.hex --force &&\ - mix local.rebar --force diff --git a/ci/elixir-1.12/build_and_push.sh b/ci/elixir-1.12/build_and_push.sh deleted file mode 100755 index 508262ed82..0000000000 --- a/ci/elixir-1.12/build_and_push.sh +++ /dev/null @@ -1 +0,0 @@ -docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.12 --push . From 1bd28e7d592b429c5eee072db8d1f2ae77d76e29 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 18 Sep 2024 17:28:48 +0000 Subject: [PATCH 057/100] CI script to build and publish an image for Elixir 1.14 --- ci/{elixir-1.13.4-otp-25 => elixir-1.14.5-otp-25}/Dockerfile | 2 +- .../build_and_push.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename ci/{elixir-1.13.4-otp-25 => elixir-1.14.5-otp-25}/Dockerfile (91%) rename ci/{elixir-1.13.4-otp-25 => elixir-1.14.5-otp-25}/build_and_push.sh (52%) diff --git a/ci/elixir-1.13.4-otp-25/Dockerfile b/ci/elixir-1.14.5-otp-25/Dockerfile similarity index 91% rename from ci/elixir-1.13.4-otp-25/Dockerfile rename to ci/elixir-1.14.5-otp-25/Dockerfile index 25a1639e89..3a35c84c39 100644 --- a/ci/elixir-1.13.4-otp-25/Dockerfile +++ b/ci/elixir-1.14.5-otp-25/Dockerfile @@ -1,4 +1,4 @@ -FROM elixir:1.13.4-otp-25 +FROM elixir:1.14.5-otp-25 # Single RUN statement, otherwise intermediate images are created # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run diff --git a/ci/elixir-1.13.4-otp-25/build_and_push.sh b/ci/elixir-1.14.5-otp-25/build_and_push.sh similarity index 52% rename from ci/elixir-1.13.4-otp-25/build_and_push.sh rename to ci/elixir-1.14.5-otp-25/build_and_push.sh index b8ca1d24d7..912c47d0c4 100755 --- a/ci/elixir-1.13.4-otp-25/build_and_push.sh +++ b/ci/elixir-1.14.5-otp-25/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.14.5-otp-25 --push . From 03e14e759db47633cce320285d93d8c1f3bde65c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:08:39 +0100 Subject: [PATCH 058/100] MRF: Add filtering against AP id --- changelog.d/mrf-id_filter.add | 1 + lib/pleroma/web/activity_pub/mrf.ex | 8 ++++++++ lib/pleroma/web/activity_pub/mrf/policy.ex | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog.d/mrf-id_filter.add diff --git a/changelog.d/mrf-id_filter.add b/changelog.d/mrf-id_filter.add new file mode 100644 index 0000000000..f556f9bc43 --- /dev/null +++ b/changelog.d/mrf-id_filter.add @@ -0,0 +1 @@ +Add `id_filter` to MRF to filter URLs and their domain prior to fetching \ No newline at end of file diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index bc418d908d..51ab476b7a 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -108,6 +108,14 @@ def filter(policies, %{} = message) do def filter(%{} = object), do: get_policies() |> filter(object) + def id_filter(policies, id) when is_binary(id) do + policies + |> Enum.filter(&function_exported?(&1, :id_filter, 1)) + |> Enum.all?(& &1.id_filter(id)) + end + + def id_filter(id) when is_binary(id), do: get_policies() |> id_filter(id) + @impl true def pipeline_filter(%{} = message, meta) do object = meta[:object_data] diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 54ca4b7357..08bcac08a6 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do @callback filter(Pleroma.Activity.t()) :: {:ok | :reject, Pleroma.Activity.t()} + @callback id_filter(String.t()) :: boolean() @callback describe() :: {:ok | :error, map()} @callback config_description() :: %{ optional(:children) => [map()], @@ -13,5 +14,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do description: String.t() } @callback history_awareness() :: :auto | :manual - @optional_callbacks config_description: 0, history_awareness: 0 + @optional_callbacks config_description: 0, history_awareness: 0, id_filter: 1 end From 3dd6f6585985a085c1c2f2243501323864dcac2d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:09:41 +0100 Subject: [PATCH 059/100] Object.Fetcher: Hook to MRF.id_filter --- lib/pleroma/object/fetcher.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 69a5f32685..c85a8b09f1 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -145,6 +145,7 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do Logger.debug("Fetching object #{id} via AP") with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")}, + {_, true} <- {:mrf, MRF.id_filter(id)}, {:ok, body} <- get_object(id), {:ok, data} <- safe_json_decode(body), :ok <- Containment.contain_origin_from_id(id, data) do @@ -160,6 +161,9 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do {:error, e} -> {:error, e} + {:mrf, false} -> + {:error, {:reject, "Filtered by id"}} + e -> {:error, e} end From 30063c5914d229753f3bacab98c38736f2a447e6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:09:58 +0100 Subject: [PATCH 060/100] MRF.DropPolicy: Add id_filter/1 --- lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index e4fcc9935b..cf07db7f30 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -13,6 +13,12 @@ def filter(activity) do {:reject, activity} end + @impl true + def id_filter(id) do + Logger.debug("REJECTING #{id}") + false + end + @impl true def describe, do: {:ok, %{}} end From 0fa13c55357ca83ae00b39626a0fa4be3a936640 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Mar 2023 09:16:25 +0100 Subject: [PATCH 061/100] MRF.SimplePolicy: Add id_filter/1 --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 12 ++++++++++++ .../web/activity_pub/mrf/simple_policy_test.exs | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index ae7f18bfe5..a97e8db7b1 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -191,6 +191,18 @@ defp instance_list(config_key) do |> MRF.instance_list_from_tuples() end + @impl true + def id_filter(id) do + host_info = URI.parse(id) + + with {:ok, _} <- check_accept(host_info, %{}), + {:ok, _} <- check_reject(host_info, %{}) do + true + else + _ -> false + end + end + @impl true def filter(%{"type" => "Delete", "actor" => actor} = activity) do %{host: actor_host} = URI.parse(actor) diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index 1a51b7d301..f49a7b8ff8 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -252,6 +252,7 @@ test "is empty" do remote_message = build_remote_message() assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(remote_message["actor"]) end test "activity has a matching host" do @@ -260,6 +261,7 @@ test "activity has a matching host" do remote_message = build_remote_message() assert {:reject, _} = SimplePolicy.filter(remote_message) + refute SimplePolicy.id_filter(remote_message["actor"]) end test "activity matches with wildcard domain" do @@ -268,6 +270,7 @@ test "activity matches with wildcard domain" do remote_message = build_remote_message() assert {:reject, _} = SimplePolicy.filter(remote_message) + refute SimplePolicy.id_filter(remote_message["actor"]) end test "actor has a matching host" do @@ -276,6 +279,7 @@ test "actor has a matching host" do remote_user = build_remote_user() assert {:reject, _} = SimplePolicy.filter(remote_user) + refute SimplePolicy.id_filter(remote_user["id"]) end test "reject Announce when object would be rejected" do @@ -288,6 +292,7 @@ test "reject Announce when object would be rejected" do } assert {:reject, _} = SimplePolicy.filter(announce) + # Note: Non-Applicable for id_filter/1 end test "reject by URI object" do @@ -300,6 +305,7 @@ test "reject by URI object" do } assert {:reject, _} = SimplePolicy.filter(announce) + # Note: Non-Applicable for id_filter/1 end end @@ -370,6 +376,8 @@ test "is empty" do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(local_message["actor"]) + assert SimplePolicy.id_filter(remote_message["actor"]) end test "is not empty but activity doesn't have a matching host" do @@ -380,6 +388,8 @@ test "is not empty but activity doesn't have a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert {:reject, _} = SimplePolicy.filter(remote_message) + assert SimplePolicy.id_filter(local_message["actor"]) + refute SimplePolicy.id_filter(remote_message["actor"]) end test "activity has a matching host" do @@ -390,6 +400,8 @@ test "activity has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(local_message["actor"]) + assert SimplePolicy.id_filter(remote_message["actor"]) end test "activity matches with wildcard domain" do @@ -400,6 +412,8 @@ test "activity matches with wildcard domain" do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + assert SimplePolicy.id_filter(local_message["actor"]) + assert SimplePolicy.id_filter(remote_message["actor"]) end test "actor has a matching host" do @@ -408,6 +422,7 @@ test "actor has a matching host" do remote_user = build_remote_user() assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + assert SimplePolicy.id_filter(remote_user["id"]) end end From a1e3fb506b309a529f0ce8ef231d853e7866be21 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 21 Sep 2024 15:39:02 +0200 Subject: [PATCH 062/100] Dockerfile: Elixir 1.14 --- Dockerfile | 7 ++++--- changelog.d/elixir-1.14-docker.skip | 0 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/elixir-1.14-docker.skip diff --git a/Dockerfile b/Dockerfile index 72461305ca..fff58154e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ +# https://hub.docker.com/r/hexpm/elixir/tags ARG ELIXIR_IMG=hexpm/elixir -ARG ELIXIR_VER=1.13.4 -ARG ERLANG_VER=24.3.4.15 -ARG ALPINE_VER=3.17.5 +ARG ELIXIR_VER=1.14.5 +ARG ERLANG_VER=25.3.2.14 +ARG ALPINE_VER=3.17.9 FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build diff --git a/changelog.d/elixir-1.14-docker.skip b/changelog.d/elixir-1.14-docker.skip new file mode 100644 index 0000000000..e69de29bb2 From 382426e0338d7918cd2db7c72ede446a2a8f7f4f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 12:41:06 -0400 Subject: [PATCH 063/100] Remove Object.get_by_id_and_maybe_refetch/2 This was only used for poll refreshing and is not a good approach to the problem. --- lib/pleroma/object.ex | 21 --- .../controllers/poll_controller.ex | 2 +- test/pleroma/object_test.exs | 144 ------------------ 3 files changed, 1 insertion(+), 166 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 748f18e6cd..77dfda8510 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -99,27 +99,6 @@ defp hashtags_changed?(_, _), do: false def get_by_id(nil), do: nil def get_by_id(id), do: Repo.get(Object, id) - @spec get_by_id_and_maybe_refetch(integer(), list()) :: Object.t() | nil - def get_by_id_and_maybe_refetch(id, opts \\ []) do - with %Object{updated_at: updated_at} = object <- get_by_id(id) do - if opts[:interval] && - NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do - case Fetcher.refetch_object(object) do - {:ok, %Object{} = object} -> - object - - e -> - Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}") - object - end - else - object - end - else - nil -> nil - end - end - def get_by_ap_id(nil), do: nil def get_by_ap_id(ap_id) do diff --git a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex index a2af8148ca..303b995f66 100644 --- a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.PollController do @doc "GET /api/v1/polls/:id" def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do - with %Object{} = object <- Object.get_by_id_and_maybe_refetch(id, interval: 60), + with %Object{} = object <- Object.get_by_id(id), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user) do try_render(conn, "show.json", %{object: object, for: user}) diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs index 48d4d86ebd..b3c528e32f 100644 --- a/test/pleroma/object_test.exs +++ b/test/pleroma/object_test.exs @@ -6,12 +6,10 @@ defmodule Pleroma.ObjectTest do use Pleroma.DataCase use Oban.Testing, repo: Pleroma.Repo - import ExUnit.CaptureLog import Mox import Pleroma.Factory import Tesla.Mock - alias Pleroma.Activity alias Pleroma.Hashtag alias Pleroma.Object alias Pleroma.Repo @@ -282,148 +280,6 @@ test "does not fetch unknown objects when fetch is false" do end end - describe "get_by_id_and_maybe_refetch" do - setup do - mock(fn - %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_original.json"), - headers: HttpRequestMock.activitypub_object_headers() - } - - env -> - apply(HttpRequestMock, :request, [env]) - end) - - mock_modified = fn resp -> - mock(fn - %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - resp - - env -> - apply(HttpRequestMock, :request, [env]) - end) - end - - on_exit(fn -> mock(fn env -> apply(HttpRequestMock, :request, [env]) end) end) - - [mock_modified: mock_modified] - end - - test "refetches if the time since the last refetch is greater than the interval", %{ - mock_modified: mock_modified - } do - %Object{} = - object = - Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d", - fetch: true - ) - - Object.set_cache(object) - - assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 - assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - - mock_modified.(%Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_modified.json"), - headers: HttpRequestMock.activitypub_object_headers() - }) - - updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) - object_in_cache = Object.get_cached_by_ap_id(object.data["id"]) - assert updated_object == object_in_cache - assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8 - assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3 - end - - test "returns the old object if refetch fails", %{mock_modified: mock_modified} do - %Object{} = - object = - Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d", - fetch: true - ) - - Object.set_cache(object) - - assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 - assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - - assert capture_log(fn -> - mock_modified.(%Tesla.Env{status: 404, body: ""}) - - updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) - object_in_cache = Object.get_cached_by_ap_id(object.data["id"]) - assert updated_object == object_in_cache - assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 - assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - end) =~ - "[error] Couldn't refresh https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d" - end - - test "does not refetch if the time since the last refetch is greater than the interval", %{ - mock_modified: mock_modified - } do - %Object{} = - object = - Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d", - fetch: true - ) - - Object.set_cache(object) - - assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 - assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - - mock_modified.(%Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_modified.json"), - headers: HttpRequestMock.activitypub_object_headers() - }) - - updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100) - object_in_cache = Object.get_cached_by_ap_id(object.data["id"]) - assert updated_object == object_in_cache - assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 - assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - end - - test "preserves internal fields on refetch", %{mock_modified: mock_modified} do - %Object{} = - object = - Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d", - fetch: true - ) - - Object.set_cache(object) - - assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 - assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - - user = insert(:user) - activity = Activity.get_create_by_object_ap_id(object.data["id"]) - {:ok, activity} = CommonAPI.favorite(activity.id, user) - object = Object.get_by_ap_id(activity.data["object"]) - - assert object.data["like_count"] == 1 - - mock_modified.(%Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_modified.json"), - headers: HttpRequestMock.activitypub_object_headers() - }) - - updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) - object_in_cache = Object.get_cached_by_ap_id(object.data["id"]) - assert updated_object == object_in_cache - assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8 - assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3 - - assert updated_object.data["like_count"] == 1 - end - end - describe ":hashtags association" do test "Hashtag records are created with Object record and updated on its change" do user = insert(:user) From 2380ae6dcc267d7d6ff81a55ae95eed718176563 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 13:38:13 -0400 Subject: [PATCH 064/100] Validate an Oban job is inserted for poll refreshes --- .../web/mastodon_api/controllers/poll_controller_test.exs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs index 7912b1d5f7..b2cceec513 100644 --- a/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.PollControllerTest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.Web.ConnCase, async: true alias Pleroma.Object @@ -27,6 +28,11 @@ test "returns poll entity for object id", %{user: user, conn: conn} do response = json_response_and_validate_schema(conn, 200) id = to_string(object.id) assert %{"id" => ^id, "expired" => false, "multiple" => false} = response + + assert_enqueued( + worker: Pleroma.Workers.PollWorker, + args: %{"op" => "refresh", "activity_id" => activity.id} + ) end test "does not expose polls for private statuses", %{conn: conn} do From c077a14ce1343f5515fa11938df7d808f23a566c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 13:54:56 -0400 Subject: [PATCH 065/100] Add Oban job to handle poll refreshing and stream out the update --- .../controllers/poll_controller.ex | 4 +++ lib/pleroma/workers/poll_worker.ex | 36 ++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex index 303b995f66..0d5a575184 100644 --- a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.PollController do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Workers.PollWorker alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug @@ -33,6 +34,9 @@ def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id} with %Object{} = object <- Object.get_by_id(id), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user) do + PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) + |> Oban.insert(unique: [period: 60]) + try_render(conn, "show.json", %{object: object, for: user}) else error when is_nil(error) or error == false -> diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index d263aa1b9e..0d2d67326a 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -11,14 +11,34 @@ defmodule Pleroma.Workers.PollWorker do alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.Object + alias Pleroma.Object.Fetcher + + @stream_out_impl Pleroma.Config.get( + [__MODULE__, :stream_out], + Pleroma.Web.ActivityPub.ActivityPub + ) @impl true def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do - with %Activity{} = activity <- find_poll_activity(activity_id), + with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(activity_id)}, {:ok, notifications} <- Notification.create_poll_notifications(activity) do Notification.stream(notifications) else - {:error, :poll_activity_not_found} = e -> {:cancel, e} + {:activity, nil} -> {:cancel, :poll_activity_not_found} + e -> {:error, e} + end + end + + def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do + with {_, %Activity{object: object}} <- + {:activity, Activity.get_by_id_with_object(activity_id)}, + {_, {:ok, _object}} <- {:refetch, Fetcher.refetch_object(object)} do + stream_update(activity_id) + + :ok + else + {:activity, nil} -> {:cancel, :poll_activity_not_found} + {:refetch, _} = e -> {:cancel, e} e -> {:error, e} end end @@ -26,12 +46,6 @@ def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do @impl true def timeout(_job), do: :timer.seconds(5) - defp find_poll_activity(activity_id) do - with nil <- Activity.get_by_id(activity_id) do - {:error, :poll_activity_not_found} - end - end - def schedule_poll_end(%Activity{data: %{"type" => "Create"}, id: activity_id} = activity) do with %Object{data: %{"type" => "Question", "closed" => closed}} when is_binary(closed) <- Object.normalize(activity), @@ -49,4 +63,10 @@ def schedule_poll_end(%Activity{data: %{"type" => "Create"}, id: activity_id} = end def schedule_poll_end(activity), do: {:error, activity} + + defp stream_update(activity_id) do + Activity.get_by_id(activity_id) + |> Activity.normalize() + |> @stream_out_impl.stream_out() + end end From 4b3f604f9529c9ced23f747cb6f6d82fedfadab0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:02:41 -0400 Subject: [PATCH 066/100] Skip refetching poll results if the object's updated_at is newer than the poll closed timestamp --- lib/pleroma/workers/poll_worker.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index 0d2d67326a..a61c5eac17 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -32,6 +32,8 @@ def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do with {_, %Activity{object: object}} <- {:activity, Activity.get_by_id_with_object(activity_id)}, + {:ok, naive_closed} <- NaiveDateTime.from_iso8601(object.data["closed"]), + {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, naive_closed)}, {_, {:ok, _object}} <- {:refetch, Fetcher.refetch_object(object)} do stream_update(activity_id) @@ -39,6 +41,7 @@ def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do else {:activity, nil} -> {:cancel, :poll_activity_not_found} {:refetch, _} = e -> {:cancel, e} + {:closed_compare, _} -> {:cancel, :poll_finalized} e -> {:error, e} end end From 47ce3a4a961bd7496f8105bc957dbf958b77d342 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:17:35 -0400 Subject: [PATCH 067/100] Schedule a final poll refresh before streaming out the notifications --- lib/pleroma/workers/poll_worker.ex | 8 ++++++-- test/pleroma/workers/poll_worker_test.exs | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index a61c5eac17..574daa9ba7 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -22,6 +22,10 @@ defmodule Pleroma.Workers.PollWorker do def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(activity_id)}, {:ok, notifications} <- Notification.create_poll_notifications(activity) do + # Schedule a final refresh + __MODULE__.new(%{"op" => "refresh", "activity_id" => activity_id}) + |> Oban.insert() + Notification.stream(notifications) else {:activity, nil} -> {:cancel, :poll_activity_not_found} @@ -32,8 +36,8 @@ def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do with {_, %Activity{object: object}} <- {:activity, Activity.get_by_id_with_object(activity_id)}, - {:ok, naive_closed} <- NaiveDateTime.from_iso8601(object.data["closed"]), - {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, naive_closed)}, + {:ok, naive_closed} <- NaiveDateTime.from_iso8601(object.data["closed"]), + {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, naive_closed)}, {_, {:ok, _object}} <- {:refetch, Fetcher.refetch_object(object)} do stream_update(activity_id) diff --git a/test/pleroma/workers/poll_worker_test.exs b/test/pleroma/workers/poll_worker_test.exs index 749df8affd..e1c67f0571 100644 --- a/test/pleroma/workers/poll_worker_test.exs +++ b/test/pleroma/workers/poll_worker_test.exs @@ -44,6 +44,12 @@ test "poll notification job" do # Ensure notifications were streamed out when job executes assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], :_)) assert called(Pleroma.Web.Push.send(:_)) + + # Ensure we scheduled a final refresh of the poll + assert_enqueued( + worker: PollWorker, + args: %{"op" => "refresh", "activity_id" => activity.id} + ) end end end From a2e7db43aa3636569f4d770df980347a03c957fe Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:23:04 -0400 Subject: [PATCH 068/100] Rename assignment for consistency --- lib/pleroma/workers/poll_worker.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index 574daa9ba7..f70ab48a4d 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -36,8 +36,8 @@ def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do with {_, %Activity{object: object}} <- {:activity, Activity.get_by_id_with_object(activity_id)}, - {:ok, naive_closed} <- NaiveDateTime.from_iso8601(object.data["closed"]), - {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, naive_closed)}, + {:ok, end_time} <- NaiveDateTime.from_iso8601(object.data["closed"]), + {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, end_time)}, {_, {:ok, _object}} <- {:refetch, Fetcher.refetch_object(object)} do stream_update(activity_id) From 766edfe5b2b19f4819704540341b8fcc92f133bd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:32:28 -0400 Subject: [PATCH 069/100] Test Poll refresh jobs stream out updates after refetching the object --- test/pleroma/workers/poll_worker_test.exs | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/pleroma/workers/poll_worker_test.exs b/test/pleroma/workers/poll_worker_test.exs index e1c67f0571..56a338bac3 100644 --- a/test/pleroma/workers/poll_worker_test.exs +++ b/test/pleroma/workers/poll_worker_test.exs @@ -52,4 +52,33 @@ test "poll notification job" do ) end end + + test "poll refresh job" do + user = insert(:user, local: false) + question = insert(:question, user: user) + activity = insert(:question_activity, question: question) + + PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) + |> Oban.insert() + + expected_job_args = %{"activity_id" => activity.id, "op" => "refresh"} + + assert_enqueued(args: expected_job_args) + + with_mocks([ + { + Pleroma.Web.Streamer, + [], + [ + stream: fn _, _ -> nil end + ] + } + ]) do + [job] = all_enqueued(worker: PollWorker) + PollWorker.perform(job) + + # Ensure updates are streamed out + assert called(Pleroma.Web.Streamer.stream(["user", "list", "public", "public:local"], :_)) + end + end end From b2340b5b776d243f6cf12971393783cc3b7c2dc2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:45:13 -0400 Subject: [PATCH 070/100] Permit backdating the poll closed timestamp --- test/support/factory.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/support/factory.ex b/test/support/factory.ex index 8f1c6faf97..732ea3143c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -241,6 +241,7 @@ def tombstone_factory do def question_factory(attrs \\ %{}) do user = attrs[:user] || insert(:user) + closed = attrs[:closed] || DateTime.utc_now() |> DateTime.add(86_400) |> DateTime.to_iso8601() data = %{ "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), @@ -251,7 +252,7 @@ def question_factory(attrs \\ %{}) do "to" => ["https://www.w3.org/ns/activitystreams#Public"], "cc" => [user.follower_address], "context" => Pleroma.Web.ActivityPub.Utils.generate_context_id(), - "closed" => DateTime.utc_now() |> DateTime.add(86_400) |> DateTime.to_iso8601(), + "closed" => closed, "content" => "Which flavor of ice cream do you prefer?", "oneOf" => [ %{ From a1b384f63c3587d0463109b74b0bbcc5c5ae82ee Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:45:41 -0400 Subject: [PATCH 071/100] Test that a poll refresh is cancelled if updated_at on the object is newer than the poll closing time --- test/pleroma/workers/poll_worker_test.exs | 61 +++++++++++++++-------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/test/pleroma/workers/poll_worker_test.exs b/test/pleroma/workers/poll_worker_test.exs index 56a338bac3..0fafcae111 100644 --- a/test/pleroma/workers/poll_worker_test.exs +++ b/test/pleroma/workers/poll_worker_test.exs @@ -53,32 +53,51 @@ test "poll notification job" do end end - test "poll refresh job" do - user = insert(:user, local: false) - question = insert(:question, user: user) - activity = insert(:question_activity, question: question) + describe "poll refresh" do + test "normal job" do + user = insert(:user, local: false) + question = insert(:question, user: user) + activity = insert(:question_activity, question: question) - PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) - |> Oban.insert() + PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) + |> Oban.insert() - expected_job_args = %{"activity_id" => activity.id, "op" => "refresh"} + expected_job_args = %{"activity_id" => activity.id, "op" => "refresh"} - assert_enqueued(args: expected_job_args) + assert_enqueued(args: expected_job_args) + + with_mocks([ + { + Pleroma.Web.Streamer, + [], + [ + stream: fn _, _ -> nil end + ] + } + ]) do + [job] = all_enqueued(worker: PollWorker) + PollWorker.perform(job) + + # Ensure updates are streamed out + assert called(Pleroma.Web.Streamer.stream(["user", "list", "public", "public:local"], :_)) + end + end + + test "when updated_at is after poll closing" do + poll_closed = DateTime.utc_now() |> DateTime.add(-86_400) |> DateTime.to_iso8601() + user = insert(:user, local: false) + question = insert(:question, user: user, closed: poll_closed) + activity = insert(:question_activity, question: question) + + PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) + |> Oban.insert() + + expected_job_args = %{"activity_id" => activity.id, "op" => "refresh"} + + assert_enqueued(args: expected_job_args) - with_mocks([ - { - Pleroma.Web.Streamer, - [], - [ - stream: fn _, _ -> nil end - ] - } - ]) do [job] = all_enqueued(worker: PollWorker) - PollWorker.perform(job) - - # Ensure updates are streamed out - assert called(Pleroma.Web.Streamer.stream(["user", "list", "public", "public:local"], :_)) + assert {:cancel, :poll_finalized} = PollWorker.perform(job) end end end From 2ab4049508148756076853bae26279b698740597 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:47:30 -0400 Subject: [PATCH 072/100] Poll refreshing changelog --- changelog.d/poll-refresh.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/poll-refresh.change diff --git a/changelog.d/poll-refresh.change b/changelog.d/poll-refresh.change new file mode 100644 index 0000000000..b755128a12 --- /dev/null +++ b/changelog.d/poll-refresh.change @@ -0,0 +1 @@ +Poll results refreshing is handled asynchronously and will not attempt to keep fetching updates to a closed poll. From b735d9e6e19a1c64f43428e6342e3d172728c736 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 14:55:38 -0400 Subject: [PATCH 073/100] Improve assertion --- test/pleroma/workers/poll_worker_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/workers/poll_worker_test.exs b/test/pleroma/workers/poll_worker_test.exs index 0fafcae111..c34647f1ba 100644 --- a/test/pleroma/workers/poll_worker_test.exs +++ b/test/pleroma/workers/poll_worker_test.exs @@ -97,7 +97,7 @@ test "when updated_at is after poll closing" do assert_enqueued(args: expected_job_args) [job] = all_enqueued(worker: PollWorker) - assert {:cancel, :poll_finalized} = PollWorker.perform(job) + assert {:cancel, :poll_finalized} == PollWorker.perform(job) end end end From 9ff57946e7d6fa7dabaf90457e11041ce46991c4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 30 Sep 2024 15:25:13 -0400 Subject: [PATCH 074/100] Credo --- lib/pleroma/web/mastodon_api/controllers/poll_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex index 0d5a575184..f89bfa7f29 100644 --- a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex @@ -9,10 +9,10 @@ defmodule Pleroma.Web.MastodonAPI.PollController do alias Pleroma.Activity alias Pleroma.Object - alias Pleroma.Workers.PollWorker alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Workers.PollWorker action_fallback(Pleroma.Web.MastodonAPI.FallbackController) From 0a42a3f2eaf53fa87d934226874de5919320de26 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 2 Oct 2024 11:05:17 -0400 Subject: [PATCH 075/100] Do not attempt to schedule poll refresh jobs for local activities --- .../controllers/poll_controller.ex | 6 ++++-- .../controllers/poll_controller_test.exs | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex index f89bfa7f29..495f89278f 100644 --- a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex @@ -34,8 +34,10 @@ def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id} with %Object{} = object <- Object.get_by_id(id), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user) do - PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) - |> Oban.insert(unique: [period: 60]) + unless activity.local do + PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) + |> Oban.insert(unique: [period: 60]) + end try_render(conn, "show.json", %{object: object, for: user}) else diff --git a/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs index b2cceec513..4b236678c0 100644 --- a/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs @@ -29,6 +29,25 @@ test "returns poll entity for object id", %{user: user, conn: conn} do id = to_string(object.id) assert %{"id" => ^id, "expired" => false, "multiple" => false} = response + refute_enqueued( + worker: Pleroma.Workers.PollWorker, + args: %{"op" => "refresh", "activity_id" => activity.id} + ) + end + + test "does not create oban job to refresh poll if activity is local", %{conn: conn} do + user = insert(:user, local: false) + question = insert(:question, user: user) + activity = insert(:question_activity, question: question, local: false) + + # Ensure this is not represented as a local activity + refute activity.local + + object = Object.normalize(activity, fetch: false) + + get(conn, "/api/v1/polls/#{object.id}") + |> json_response_and_validate_schema(200) + assert_enqueued( worker: Pleroma.Workers.PollWorker, args: %{"op" => "refresh", "activity_id" => activity.id} From ba2ae5e40bbe98d20be083d331222a9aea8b61de Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 3 Oct 2024 10:14:02 -0400 Subject: [PATCH 076/100] Check if a refresh is permitted by comparing timestamps before attempting to insert an Oban job It's better to avoid inserting an Oban job that will just be rejected if it's not expensive to check. --- .../mastodon_api/controllers/poll_controller.ex | 17 ++++++++++++----- lib/pleroma/workers/poll_worker.ex | 3 --- .../controllers/poll_controller_test.exs | 5 ++++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex index 495f89278f..4b347a6a7b 100644 --- a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex @@ -32,12 +32,10 @@ defmodule Pleroma.Web.MastodonAPI.PollController do @doc "GET /api/v1/polls/:id" def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do with %Object{} = object <- Object.get_by_id(id), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + %Activity{} = activity <- + Activity.get_create_by_object_ap_id_with_object(object.data["id"]), true <- Visibility.visible_for_user?(activity, user) do - unless activity.local do - PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) - |> Oban.insert(unique: [period: 60]) - end + maybe_refresh_poll(activity) try_render(conn, "show.json", %{object: object, for: user}) else @@ -76,4 +74,13 @@ defp get_cached_vote_or_vote(object, user, choices) do end end) end + + defp maybe_refresh_poll(%Activity{object: %Object{} = object} = activity) do + with false <- activity.local, + {:ok, end_time} <- NaiveDateTime.from_iso8601(object.data["closed"]), + {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, end_time)} do + PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) + |> Oban.insert(unique: [period: 60]) + end + end end diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index f70ab48a4d..bb92634c98 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -36,8 +36,6 @@ def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do with {_, %Activity{object: object}} <- {:activity, Activity.get_by_id_with_object(activity_id)}, - {:ok, end_time} <- NaiveDateTime.from_iso8601(object.data["closed"]), - {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, end_time)}, {_, {:ok, _object}} <- {:refetch, Fetcher.refetch_object(object)} do stream_update(activity_id) @@ -45,7 +43,6 @@ def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do else {:activity, nil} -> {:cancel, :poll_activity_not_found} {:refetch, _} = e -> {:cancel, e} - {:closed_compare, _} -> {:cancel, :poll_finalized} e -> {:error, e} end end diff --git a/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs index 4b236678c0..51af877424 100644 --- a/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs @@ -29,13 +29,16 @@ test "returns poll entity for object id", %{user: user, conn: conn} do id = to_string(object.id) assert %{"id" => ^id, "expired" => false, "multiple" => false} = response + # Local activities should not generate an Oban job to refresh + assert activity.local + refute_enqueued( worker: Pleroma.Workers.PollWorker, args: %{"op" => "refresh", "activity_id" => activity.id} ) end - test "does not create oban job to refresh poll if activity is local", %{conn: conn} do + test "creates an oban job to refresh poll if activity is remote", %{conn: conn} do user = insert(:user, local: false) question = insert(:question, user: user) activity = insert(:question_activity, question: question, local: false) From fa8de790dfbdb2cc7de212be4ecdd2823048ba8f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 3 Oct 2024 10:19:10 -0400 Subject: [PATCH 077/100] Remove test superceded by logic change We will not be inserting jobs that should be skipped due to updated_at --- test/pleroma/workers/poll_worker_test.exs | 61 ++++++++--------------- 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/test/pleroma/workers/poll_worker_test.exs b/test/pleroma/workers/poll_worker_test.exs index c34647f1ba..70eb7c4226 100644 --- a/test/pleroma/workers/poll_worker_test.exs +++ b/test/pleroma/workers/poll_worker_test.exs @@ -53,51 +53,32 @@ test "poll notification job" do end end - describe "poll refresh" do - test "normal job" do - user = insert(:user, local: false) - question = insert(:question, user: user) - activity = insert(:question_activity, question: question) + test "poll refresh" do + user = insert(:user, local: false) + question = insert(:question, user: user) + activity = insert(:question_activity, question: question) - PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) - |> Oban.insert() + PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) + |> Oban.insert() - expected_job_args = %{"activity_id" => activity.id, "op" => "refresh"} + expected_job_args = %{"activity_id" => activity.id, "op" => "refresh"} - assert_enqueued(args: expected_job_args) - - with_mocks([ - { - Pleroma.Web.Streamer, - [], - [ - stream: fn _, _ -> nil end - ] - } - ]) do - [job] = all_enqueued(worker: PollWorker) - PollWorker.perform(job) - - # Ensure updates are streamed out - assert called(Pleroma.Web.Streamer.stream(["user", "list", "public", "public:local"], :_)) - end - end - - test "when updated_at is after poll closing" do - poll_closed = DateTime.utc_now() |> DateTime.add(-86_400) |> DateTime.to_iso8601() - user = insert(:user, local: false) - question = insert(:question, user: user, closed: poll_closed) - activity = insert(:question_activity, question: question) - - PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) - |> Oban.insert() - - expected_job_args = %{"activity_id" => activity.id, "op" => "refresh"} - - assert_enqueued(args: expected_job_args) + assert_enqueued(args: expected_job_args) + with_mocks([ + { + Pleroma.Web.Streamer, + [], + [ + stream: fn _, _ -> nil end + ] + } + ]) do [job] = all_enqueued(worker: PollWorker) - assert {:cancel, :poll_finalized} == PollWorker.perform(job) + PollWorker.perform(job) + + # Ensure updates are streamed out + assert called(Pleroma.Web.Streamer.stream(["user", "list", "public", "public:local"], :_)) end end end From b854e3836fd22a2589a6a6b97478998675d72048 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 3 Oct 2024 10:30:32 -0400 Subject: [PATCH 078/100] Remove pattern that can never match --- lib/pleroma/workers/poll_worker.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index bb92634c98..7d69bea548 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -43,7 +43,6 @@ def perform(%Job{args: %{"op" => "refresh", "activity_id" => activity_id}}) do else {:activity, nil} -> {:cancel, :poll_activity_not_found} {:refetch, _} = e -> {:cancel, e} - e -> {:error, e} end end From a3038aa6a2189ced1e5c394a4e6e8be76f2644d0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 3 Oct 2024 11:01:33 -0400 Subject: [PATCH 079/100] Increase poll refresh interval to 120 seconds --- lib/pleroma/web/mastodon_api/controllers/poll_controller.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex index 4b347a6a7b..6526457df3 100644 --- a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex @@ -28,6 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.PollController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PollOperation @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @poll_refresh_interval 120 @doc "GET /api/v1/polls/:id" def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do @@ -80,7 +81,7 @@ defp maybe_refresh_poll(%Activity{object: %Object{} = object} = activity) do {:ok, end_time} <- NaiveDateTime.from_iso8601(object.data["closed"]), {_, :lt} <- {:closed_compare, NaiveDateTime.compare(object.updated_at, end_time)} do PollWorker.new(%{"op" => "refresh", "activity_id" => activity.id}) - |> Oban.insert(unique: [period: 60]) + |> Oban.insert(unique: [period: @poll_refresh_interval]) end end end From 4533f171ab5b73e5fc332c8f65fcf1e39e4d6003 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 5 Nov 2022 13:56:56 -0500 Subject: [PATCH 080/100] Add RemoteReportPolicy to reject reports without enough information --- config/config.exs | 4 + .../activity_pub/mrf/remote_report_policy.ex | 80 +++++++++++++++++ .../mrf/remote_report_policy_test.exs | 85 +++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex create mode 100644 test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs diff --git a/config/config.exs b/config/config.exs index 47ddfac5ac..203a61c758 100644 --- a/config/config.exs +++ b/config/config.exs @@ -434,6 +434,10 @@ config :pleroma, :mrf_inline_quote, template: "RT: {url}" +config :pleroma, :mrf_remote_report, + reject_anonymous: true, + reject_empty_message: true + config :pleroma, :mrf_force_mention, mention_parent: true, mention_quoted: true diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex new file mode 100644 index 0000000000..3cf47e3ed8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -0,0 +1,80 @@ +defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do + @moduledoc "Drop remote reports if they don't contain enough information." + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Config + + @impl true + def filter(%{"type" => "Flag"} = object) do + with {_, false} <- {:local, local?(object)}, + {:ok, _} <- maybe_reject_anonymous(object), + {:ok, _} <- maybe_reject_empty_message(object) do + {:ok, object} + else + {:local, true} -> {:ok, object} + {:reject, message} -> {:reject, message} + error -> {:reject, error} + end + end + + def filter(object), do: {:ok, object} + + defp maybe_reject_anonymous(%{"actor" => actor} = object) do + with true <- Config.get([:mrf_remote_report, :reject_anonymous]), + %URI{path: "/actor"} <- URI.parse(actor) do + {:reject, "[RemoteReportPolicy] Anonymous: #{actor}"} + else + _ -> {:ok, object} + end + end + + defp maybe_reject_empty_message(%{"content" => content} = object) + when is_binary(content) and content != "" do + {:ok, object} + end + + defp maybe_reject_empty_message(object) do + if Config.get([:mrf_remote_report, :reject_empty_message]) do + {:reject, ["RemoteReportPolicy] No content"]} + else + {:ok, object} + end + end + + defp local?(%{"actor" => actor}) do + String.starts_with?(actor, Pleroma.Web.Endpoint.url()) + end + + @impl true + def describe do + mrf_remote_report = + Config.get(:mrf_remote_report) + |> Enum.into(%{}) + + {:ok, %{mrf_remote_report: mrf_remote_report}} + end + + @impl true + def config_description do + %{ + key: :mrf_remote_report, + related_policy: "Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy", + label: "MRF Remote Report", + description: "Drop remote reports if they don't contain enough information.", + children: [ + %{ + key: :reject_anonymous, + type: :boolean, + description: "Reject anonymous remote reports?", + suggestions: [true] + }, + %{ + key: :reject_empty_message, + type: :boolean, + description: "Reject remote reports with no message?", + suggestions: [true] + } + ] + } + end +end diff --git a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs new file mode 100644 index 0000000000..55fa0f2f20 --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs @@ -0,0 +1,85 @@ +defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicyTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy + + test "doesn't impact local report" do + clear_config([:mrf_remote_report, :reject_anonymous], true) + clear_config([:mrf_remote_report, :reject_empty_message], true) + + activity = %{ + "type" => "Flag", + "actor" => "http://localhost:4001/actor" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + + test "rejects anonymous report if `reject_anonymous: true`" do + clear_config([:mrf_remote_report, :reject_anonymous], true) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/actor" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end + + test "preserves anonymous report if `reject_anonymous: false`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/actor" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + + test "rejects empty message report if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_empty_message], true) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end + + test "rejects empty message report (\"\") if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_empty_message], true) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron", + "content" => "" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end + + test "preserves empty message report if `reject_empty_message: false`" do + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + + test "preserves anonymous, empty message report with all settings disabled" do + clear_config([:mrf_remote_report, :reject_empty_message], false) + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/actor" + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end +end From b7c91876d2cc027a5a7f8a79ba256f13af623997 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 5 Nov 2022 14:07:37 -0500 Subject: [PATCH 081/100] RemoteReportPolicy: add `:reject_all` option, fix tests --- config/config.exs | 1 + .../activity_pub/mrf/remote_report_policy.ex | 15 +++++++++++ .../mrf/remote_report_policy_test.exs | 25 ++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 203a61c758..07e98011d0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -435,6 +435,7 @@ config :pleroma, :mrf_inline_quote, template: "RT: {url}" config :pleroma, :mrf_remote_report, + reject_all: false, reject_anonymous: true, reject_empty_message: true diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index 3cf47e3ed8..0bd83d8f00 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do @impl true def filter(%{"type" => "Flag"} = object) do with {_, false} <- {:local, local?(object)}, + {:ok, _} <- maybe_reject_all(object), {:ok, _} <- maybe_reject_anonymous(object), {:ok, _} <- maybe_reject_empty_message(object) do {:ok, object} @@ -19,6 +20,14 @@ def filter(%{"type" => "Flag"} = object) do def filter(object), do: {:ok, object} + defp maybe_reject_all(object) do + if Config.get([:mrf_remote_report, :reject_all]) do + {:reject, "[RemoteReportPolicy] Remote report"} + else + {:ok, object} + end + end + defp maybe_reject_anonymous(%{"actor" => actor} = object) do with true <- Config.get([:mrf_remote_report, :reject_anonymous]), %URI{path: "/actor"} <- URI.parse(actor) do @@ -62,6 +71,12 @@ def config_description do label: "MRF Remote Report", description: "Drop remote reports if they don't contain enough information.", children: [ + %{ + key: :reject_all, + type: :boolean, + description: "Reject all remote reports? (this option takes precedence)", + suggestions: [false] + }, %{ key: :reject_anonymous, type: :boolean, diff --git a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs index 55fa0f2f20..43258a7f60 100644 --- a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs @@ -3,6 +3,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicyTest do alias Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy + setup do + clear_config([:mrf_remote_report, :reject_all], false) + end + test "doesn't impact local report" do clear_config([:mrf_remote_report, :reject_anonymous], true) clear_config([:mrf_remote_report, :reject_empty_message], true) @@ -17,6 +21,7 @@ test "doesn't impact local report" do test "rejects anonymous report if `reject_anonymous: true`" do clear_config([:mrf_remote_report, :reject_anonymous], true) + clear_config([:mrf_remote_report, :reject_empty_message], true) activity = %{ "type" => "Flag", @@ -28,6 +33,7 @@ test "rejects anonymous report if `reject_anonymous: true`" do test "preserves anonymous report if `reject_anonymous: false`" do clear_config([:mrf_remote_report, :reject_anonymous], false) + clear_config([:mrf_remote_report, :reject_empty_message], false) activity = %{ "type" => "Flag", @@ -38,6 +44,7 @@ test "preserves anonymous report if `reject_anonymous: false`" do end test "rejects empty message report if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], true) activity = %{ @@ -49,6 +56,7 @@ test "rejects empty message report if `reject_empty_message: true`" do end test "rejects empty message report (\"\") if `reject_empty_message: true`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], true) activity = %{ @@ -61,6 +69,7 @@ test "rejects empty message report (\"\") if `reject_empty_message: true`" do end test "preserves empty message report if `reject_empty_message: false`" do + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], false) activity = %{ @@ -72,7 +81,7 @@ test "preserves empty message report if `reject_empty_message: false`" do end test "preserves anonymous, empty message report with all settings disabled" do - clear_config([:mrf_remote_report, :reject_empty_message], false) + clear_config([:mrf_remote_report, :reject_anonymous], false) clear_config([:mrf_remote_report, :reject_empty_message], false) activity = %{ @@ -82,4 +91,18 @@ test "preserves anonymous, empty message report with all settings disabled" do assert {:ok, _} = RemoteReportPolicy.filter(activity) end + + test "reject remote report if `reject_all: true`" do + clear_config([:mrf_remote_report, :reject_all], true) + clear_config([:mrf_remote_report, :reject_anonymous], false) + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron", + "content" => "Transphobia" + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end end From fd83b86b99ee6642fa0a765a55c0f0e35f272151 Mon Sep 17 00:00:00 2001 From: Mint Date: Tue, 12 Mar 2024 22:45:15 +0300 Subject: [PATCH 082/100] RemoteReportPolicy: add `reject_third_party` option --- .../activity_pub/mrf/remote_report_policy.ex | 22 +++++++++ .../mrf/remote_report_policy_test.exs | 48 ++++++++++++++++--- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index 0bd83d8f00..964c59cbfe 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -9,6 +9,7 @@ def filter(%{"type" => "Flag"} = object) do with {_, false} <- {:local, local?(object)}, {:ok, _} <- maybe_reject_all(object), {:ok, _} <- maybe_reject_anonymous(object), + {:ok, _} <- maybe_reject_third_party(object), {:ok, _} <- maybe_reject_empty_message(object) do {:ok, object} else @@ -37,6 +38,21 @@ defp maybe_reject_anonymous(%{"actor" => actor} = object) do end end + defp maybe_reject_third_party(%{"object" => objects} = object) do + {_, to} = case objects do + [head | tail] when is_binary(head) -> {tail, head} + s when is_binary(s) -> {[], s} + _ -> {[], ""} + end + + with true <- Config.get([:mrf_remote_report, :reject_third_party]), + String.starts_with?(to, Pleroma.Web.Endpoint.url()) do + {:reject, "[RemoteReportPolicy] Third-party: #{to}"} + else + _ -> {:ok, object} + end + end + defp maybe_reject_empty_message(%{"content" => content} = object) when is_binary(content) and content != "" do {:ok, object} @@ -83,6 +99,12 @@ def config_description do description: "Reject anonymous remote reports?", suggestions: [true] }, + %{ + key: :reject_third_party, + type: :boolean, + description: "Reject reports on users from third-party instances?", + suggestions: [true] + }, %{ key: :reject_empty_message, type: :boolean, diff --git a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs index 43258a7f60..dd56a1e9b9 100644 --- a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs @@ -13,7 +13,8 @@ test "doesn't impact local report" do activity = %{ "type" => "Flag", - "actor" => "http://localhost:4001/actor" + "actor" => "http://localhost:4001/actor", + "object" => ["https://mastodon.online/users/Gargron"] } assert {:ok, _} = RemoteReportPolicy.filter(activity) @@ -25,7 +26,8 @@ test "rejects anonymous report if `reject_anonymous: true`" do activity = %{ "type" => "Flag", - "actor" => "https://mastodon.social/actor" + "actor" => "https://mastodon.social/actor", + "object" => ["https://mastodon.online/users/Gargron"] } assert {:reject, _} = RemoteReportPolicy.filter(activity) @@ -37,7 +39,34 @@ test "preserves anonymous report if `reject_anonymous: false`" do activity = %{ "type" => "Flag", - "actor" => "https://mastodon.social/actor" + "actor" => "https://mastodon.social/actor", + "object" => ["https://mastodon.online/users/Gargron"] + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + + test "rejects report on third-party if `reject_third_party: true`" do + clear_config([:mrf_remote_report, :reject_third_party], true) + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron", + "object" => ["https://mastodon.online/users/Gargron"] + } + + assert {:reject, _} = RemoteReportPolicy.filter(activity) + end + + test "preserves report on third party if `reject_third_party: false`" do + clear_config([:mrf_remote_report, :reject_third_party], false) + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron", + "object" => ["https://mastodon.online/users/Gargron"] } assert {:ok, _} = RemoteReportPolicy.filter(activity) @@ -49,7 +78,8 @@ test "rejects empty message report if `reject_empty_message: true`" do activity = %{ "type" => "Flag", - "actor" => "https://mastodon.social/users/Gargron" + "actor" => "https://mastodon.social/users/Gargron", + "object" => ["https://mastodon.online/users/Gargron"] } assert {:reject, _} = RemoteReportPolicy.filter(activity) @@ -62,6 +92,7 @@ test "rejects empty message report (\"\") if `reject_empty_message: true`" do activity = %{ "type" => "Flag", "actor" => "https://mastodon.social/users/Gargron", + "object" => ["https://mastodon.online/users/Gargron"], "content" => "" } @@ -74,7 +105,8 @@ test "preserves empty message report if `reject_empty_message: false`" do activity = %{ "type" => "Flag", - "actor" => "https://mastodon.social/users/Gargron" + "actor" => "https://mastodon.social/users/Gargron", + "object" => ["https://mastodon.online/users/Gargron"] } assert {:ok, _} = RemoteReportPolicy.filter(activity) @@ -86,7 +118,8 @@ test "preserves anonymous, empty message report with all settings disabled" do activity = %{ "type" => "Flag", - "actor" => "https://mastodon.social/actor" + "actor" => "https://mastodon.social/actor", + "object" => ["https://mastodon.online/users/Gargron"] } assert {:ok, _} = RemoteReportPolicy.filter(activity) @@ -100,7 +133,8 @@ test "reject remote report if `reject_all: true`" do activity = %{ "type" => "Flag", "actor" => "https://mastodon.social/users/Gargron", - "content" => "Transphobia" + "content" => "Transphobia", + "object" => ["https://mastodon.online/users/Gargron"] } assert {:reject, _} = RemoteReportPolicy.filter(activity) From 55612cb8ee4908a2fbb200ff581bb07c7e43410a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 12 Mar 2024 15:52:33 -0500 Subject: [PATCH 083/100] mix format --- .../web/activity_pub/mrf/remote_report_policy.ex | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index 964c59cbfe..d330289310 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -39,11 +39,12 @@ defp maybe_reject_anonymous(%{"actor" => actor} = object) do end defp maybe_reject_third_party(%{"object" => objects} = object) do - {_, to} = case objects do - [head | tail] when is_binary(head) -> {tail, head} - s when is_binary(s) -> {[], s} - _ -> {[], ""} - end + {_, to} = + case objects do + [head | tail] when is_binary(head) -> {tail, head} + s when is_binary(s) -> {[], s} + _ -> {[], ""} + end with true <- Config.get([:mrf_remote_report, :reject_third_party]), String.starts_with?(to, Pleroma.Web.Endpoint.url()) do From 48af6850fc2903d6f8c7cbf43b7db6b769c37a2a Mon Sep 17 00:00:00 2001 From: Mint Date: Fri, 12 Apr 2024 23:04:37 +0300 Subject: [PATCH 084/100] RemoteReportPolicy: Fix third-party report detection --- .../web/activity_pub/mrf/remote_report_policy.ex | 2 +- .../mrf/remote_report_policy_test.exs | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index d330289310..fa0610bf10 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -47,7 +47,7 @@ defp maybe_reject_third_party(%{"object" => objects} = object) do end with true <- Config.get([:mrf_remote_report, :reject_third_party]), - String.starts_with?(to, Pleroma.Web.Endpoint.url()) do + false <- String.starts_with?(to, Pleroma.Web.Endpoint.url()) do {:reject, "[RemoteReportPolicy] Third-party: #{to}"} else _ -> {:ok, object} diff --git a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs index dd56a1e9b9..8d2a6b4fa9 100644 --- a/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/remote_report_policy_test.exs @@ -46,7 +46,7 @@ test "preserves anonymous report if `reject_anonymous: false`" do assert {:ok, _} = RemoteReportPolicy.filter(activity) end - test "rejects report on third-party if `reject_third_party: true`" do + test "rejects report on third party if `reject_third_party: true`" do clear_config([:mrf_remote_report, :reject_third_party], true) clear_config([:mrf_remote_report, :reject_empty_message], false) @@ -59,6 +59,19 @@ test "rejects report on third-party if `reject_third_party: true`" do assert {:reject, _} = RemoteReportPolicy.filter(activity) end + test "preserves report on first party if `reject_third_party: true`" do + clear_config([:mrf_remote_report, :reject_third_party], true) + clear_config([:mrf_remote_report, :reject_empty_message], false) + + activity = %{ + "type" => "Flag", + "actor" => "https://mastodon.social/users/Gargron", + "object" => ["http://localhost:4001/actor"] + } + + assert {:ok, _} = RemoteReportPolicy.filter(activity) + end + test "preserves report on third party if `reject_third_party: false`" do clear_config([:mrf_remote_report, :reject_third_party], false) clear_config([:mrf_remote_report, :reject_empty_message], false) From eb971aa022f524f364daf24d6e6d617bfc5ca036 Mon Sep 17 00:00:00 2001 From: Mint Date: Thu, 3 Oct 2024 20:02:58 +0300 Subject: [PATCH 085/100] Changelog --- changelog.d/remote-report-policy.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/remote-report-policy.add diff --git a/changelog.d/remote-report-policy.add b/changelog.d/remote-report-policy.add new file mode 100644 index 0000000000..1cf25b1a8f --- /dev/null +++ b/changelog.d/remote-report-policy.add @@ -0,0 +1 @@ +Added RemoteReportPolicy from Rebased for handling bogus federated reports From fe4399022668df08f529af293fe5093531ebccb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 3 Oct 2024 20:27:19 +0200 Subject: [PATCH 086/100] bump postgresql version for CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .github/workflows/pl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pl.yaml b/.github/workflows/pl.yaml index 58cbb980db..10dc9369fb 100644 --- a/.github/workflows/pl.yaml +++ b/.github/workflows/pl.yaml @@ -20,7 +20,7 @@ jobs: test: services: db: - image: postgres:12 + image: postgres:16 ports: ['5432:5432'] env: POSTGRES_DB: pleroma_test From 91c7d7914b0782261d512fc2199bbd48cd1f3a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 3 Oct 2024 20:59:56 +0200 Subject: [PATCH 087/100] Fix readability issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/hashtag.ex | 2 +- lib/pleroma/user.ex | 6 +++--- lib/pleroma/user/hashtag_follow.ex | 2 +- .../web/api_spec/operations/akkoma_compat_operation.ex | 4 ++-- .../mastodon_api/controllers/follow_request_controller.ex | 2 +- lib/pleroma/web/mastodon_api/controllers/tag_controller.ex | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 29e95e3a03..3682f0c140 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -10,9 +10,9 @@ defmodule Pleroma.Hashtag do alias Ecto.Multi alias Pleroma.Hashtag - alias Pleroma.User.HashtagFollow alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.User.HashtagFollow schema "hashtags" do field(:name, :string) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2141948fc0..d5c8f2bf38 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -21,7 +21,6 @@ defmodule Pleroma.User do alias Pleroma.FollowingRelationship alias Pleroma.Formatter alias Pleroma.Hashtag - alias Pleroma.User.HashtagFollow alias Pleroma.HTML alias Pleroma.Keys alias Pleroma.MFA @@ -31,6 +30,7 @@ defmodule Pleroma.User do alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship + alias Pleroma.User.HashtagFollow alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline @@ -1167,7 +1167,7 @@ def needs_update?(_), do: true # "Locked" (self-locked) users demand explicit authorization of follow requests @spec can_direct_follow_local(User.t(), User.t()) :: true | false def can_direct_follow_local(%User{} = follower, %User{local: true} = followed) do - !followed.is_locked || (followed.permit_followback and is_friend_of(follower, followed)) + !followed.is_locked || (followed.permit_followback and friend_of?(follower, followed)) end @spec maybe_direct_follow(User.t(), User.t()) :: @@ -1552,7 +1552,7 @@ def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) |> Repo.all() end - def is_friend_of(%User{} = potential_friend, %User{local: true} = user) do + def friend_of?(%User{} = potential_friend, %User{local: true} = user) do user |> get_friends_query() |> where(id: ^potential_friend.id) diff --git a/lib/pleroma/user/hashtag_follow.ex b/lib/pleroma/user/hashtag_follow.ex index dd0254ef4c..3e28b130b4 100644 --- a/lib/pleroma/user/hashtag_follow.ex +++ b/lib/pleroma/user/hashtag_follow.ex @@ -3,9 +3,9 @@ defmodule Pleroma.User.HashtagFollow do import Ecto.Query import Ecto.Changeset - alias Pleroma.User alias Pleroma.Hashtag alias Pleroma.Repo + alias Pleroma.User schema "user_follows_hashtag" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) diff --git a/lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex b/lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex index 83d29caf9b..3f38e54e64 100644 --- a/lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/akkoma_compat_operation.ex @@ -13,7 +13,7 @@ def open_api_operation(action) do end # Adapted from https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/lib/pleroma/web/api_spec/operations/translate_operation.ex - def translation_languages_operation() do + def translation_languages_operation do %Operation{ tags: ["Akkoma compatibility routes"], summary: "Get translation languages", @@ -54,7 +54,7 @@ defp languages_schema do } end - def translate_operation() do + def translate_operation do %Operation{ tags: ["Akkoma compatibility routes"], summary: "Translate status", diff --git a/lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex b/lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex index 46f07f3bf1..a15029d92d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Web.MastodonAPI.FollowRequestController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + alias Pleroma.Pagination alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug - alias Pleroma.Pagination plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) plug(:assign_follower when action != :index) diff --git a/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex b/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex index ca5ee48ac1..21c21e984d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex @@ -2,9 +2,9 @@ defmodule Pleroma.Web.MastodonAPI.TagController do @moduledoc "Hashtag routes for mastodon API" use Pleroma.Web, :controller - alias Pleroma.User alias Pleroma.Hashtag alias Pleroma.Pagination + alias Pleroma.User import Pleroma.Web.ControllerHelper, only: [ From 84b43682112b2e19d0e37fed44cb01ea5dc0d781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 6 Oct 2024 17:00:39 +0200 Subject: [PATCH 088/100] Metadata: Do not include .atom feed links for remote accounts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/atom-tag.change | 1 + lib/pleroma/web/metadata/providers/feed.ex | 4 +++- test/pleroma/web/metadata/providers/feed_test.exs | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 changelog.d/atom-tag.change diff --git a/changelog.d/atom-tag.change b/changelog.d/atom-tag.change new file mode 100644 index 0000000000..1b3590dea8 --- /dev/null +++ b/changelog.d/atom-tag.change @@ -0,0 +1 @@ +Metadata: Do not include .atom feed links for remote accounts diff --git a/lib/pleroma/web/metadata/providers/feed.ex b/lib/pleroma/web/metadata/providers/feed.ex index e97d6a54f0..3811f96f63 100644 --- a/lib/pleroma/web/metadata/providers/feed.ex +++ b/lib/pleroma/web/metadata/providers/feed.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Metadata.Providers.Feed do @behaviour Provider @impl Provider - def build_tags(%{user: user}) do + def build_tags(%{user: %{local: true} = user}) do [ {:link, [ @@ -20,4 +20,6 @@ def build_tags(%{user: user}) do ], []} ] end + + def build_tags(_), do: [] end diff --git a/test/pleroma/web/metadata/providers/feed_test.exs b/test/pleroma/web/metadata/providers/feed_test.exs index e593453dad..40d9d09090 100644 --- a/test/pleroma/web/metadata/providers/feed_test.exs +++ b/test/pleroma/web/metadata/providers/feed_test.exs @@ -15,4 +15,10 @@ test "it renders a link to user's atom feed" do [rel: "alternate", type: "application/atom+xml", href: "/users/lain/feed.atom"], []} ] end + + test "it doesn't render a link to remote user's feed" do + user = insert(:user, nickname: "lain@lain.com", local: false) + + assert Feed.build_tags(%{user: user}) == [] + end end From 6fe062ed6c40748610e6445cec1a9878de626e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 8 Oct 2024 23:49:12 +0200 Subject: [PATCH 089/100] Add replacement for $ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/activity_pub/transmogrifier.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3a0773fc4f..4ecdb19e1b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1020,6 +1020,7 @@ defp replace_instance_host(value, nil), do: value defp replace_instance_host(content, host) when is_binary(content) do content |> String.replace("$INSTANCE$host$", host) + |> String.replace("$INSTANCE$tld$", get_tld(host, "$INSTANCE$tld$")) end defp replace_instance_host(object, host) when is_map(object) do @@ -1031,6 +1032,14 @@ defp replace_instance_host(object, host) when is_map(object) do defp replace_instance_host(value, _), do: value + defp get_tld(host, default) do + with [domain | _] <- String.split(host, ".") |> Enum.reverse() do + domain + else + _ -> default + end + end + defp patch_content_map(%{"contentMap" => %{} = content_map}, host) do content_map |> Enum.map(fn {key, value} -> {key, replace_instance_host(value, host)} end) From f758b6e37c80f5adeba74009e1cc72a420937a30 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 8 Oct 2024 23:09:59 -0400 Subject: [PATCH 090/100] Fix incoming Blocks being rejected --- changelog.d/incoming-blocks.fix | 1 + lib/pleroma/constants.ex | 5 +++++ .../web/activity_pub/object_validator.ex | 12 +++++++++++ .../activity_pub_controller_test.exs | 21 +++++++++++++++++++ 4 files changed, 39 insertions(+) create mode 100644 changelog.d/incoming-blocks.fix diff --git a/changelog.d/incoming-blocks.fix b/changelog.d/incoming-blocks.fix new file mode 100644 index 0000000000..3228d7318c --- /dev/null +++ b/changelog.d/incoming-blocks.fix @@ -0,0 +1 @@ +Fix incoming Block activities being rejected diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 5268ebe7a2..2828c79a92 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -87,6 +87,7 @@ defmodule Pleroma.Constants do const(activity_types, do: [ + "Block", "Create", "Update", "Delete", @@ -115,6 +116,10 @@ defmodule Pleroma.Constants do ] ) + const(object_types, + do: ~w[Event Question Answer Audio Video Image Article Note Page ChatMessage] + ) + # basic regex, just there to weed out potential mistakes # https://datatracker.ietf.org/doc/html/rfc2045#section-5.1 const(mime_regex, diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index b3043b93ad..35774d4107 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do @behaviour Pleroma.Web.ActivityPub.ObjectValidator.Validating + import Pleroma.Constants, only: [activity_types: 0, object_types: 0] + alias Pleroma.Activity alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object @@ -38,6 +40,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do @impl true def validate(object, meta) + # This overload works together with the InboxGuardPlug + # and ensures that we are not accepting any activity type + # that cannot pass InboxGuardPlug. + # If we want to support any more activity types, make sure to + # add it in Pleroma.Constants's activity_types or object_types, + # and, if applicable, allowed_activity_types_from_strangers. + def validate(%{"type" => type}, _meta) + when type not in activity_types() and type not in object_types(), + do: {:error, :not_allowed_object_type} + def validate(%{"type" => "Block"} = block_activity, meta) do with {:ok, block_activity} <- block_activity diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 3bd589f490..d4175b56fd 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -1320,6 +1320,27 @@ test "forwarded report from mastodon", %{conn: conn} do html_body: ~r/#{note.data["object"]}/i ) end + + test "it accepts an incoming Block", %{conn: conn, data: data} do + user = insert(:user) + + data = + data + |> Map.put("type", "Block") + |> Map.put("to", [user.ap_id]) + |> Map.put("cc", []) + |> Map.put("object", user.ap_id) + + conn = + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/users/#{user.nickname}/inbox", data) + + assert "ok" == json_response(conn, 200) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + assert Activity.get_by_ap_id(data["id"]) + end end describe "GET /users/:nickname/outbox" do From 03a6e33b81281256f2e9b6ffb75910fdd1a7894f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 9 Oct 2024 16:25:58 -0400 Subject: [PATCH 091/100] Skip the final refresh job if the activity is local --- lib/pleroma/workers/poll_worker.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index 7d69bea548..a9afe9d63a 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -22,9 +22,11 @@ defmodule Pleroma.Workers.PollWorker do def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(activity_id)}, {:ok, notifications} <- Notification.create_poll_notifications(activity) do - # Schedule a final refresh - __MODULE__.new(%{"op" => "refresh", "activity_id" => activity_id}) - |> Oban.insert() + unless activity.local do + # Schedule a final refresh + __MODULE__.new(%{"op" => "refresh", "activity_id" => activity_id}) + |> Oban.insert() + end Notification.stream(notifications) else From 5b04c2bf131f70120c407f5b4c242e3d245151f8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 9 Oct 2024 20:15:00 -0400 Subject: [PATCH 092/100] Test the final refresh behavior of a PollWorker poll_end job --- test/pleroma/workers/poll_worker_test.exs | 32 ++++++++++++++++++++--- test/support/factory.ex | 3 ++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/test/pleroma/workers/poll_worker_test.exs b/test/pleroma/workers/poll_worker_test.exs index 70eb7c4226..a7cbbdb83c 100644 --- a/test/pleroma/workers/poll_worker_test.exs +++ b/test/pleroma/workers/poll_worker_test.exs @@ -11,10 +11,10 @@ defmodule Pleroma.Workers.PollWorkerTest do alias Pleroma.Workers.PollWorker - test "poll notification job" do + test "local poll ending notification job" do user = insert(:user) question = insert(:question, user: user) - activity = insert(:question_activity, question: question) + activity = insert(:question_activity, question: question, user: user) PollWorker.schedule_poll_end(activity) @@ -45,14 +45,38 @@ test "poll notification job" do assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], :_)) assert called(Pleroma.Web.Push.send(:_)) - # Ensure we scheduled a final refresh of the poll - assert_enqueued( + # Skip refreshing polls for local activities + assert activity.local + + refute_enqueued( worker: PollWorker, args: %{"op" => "refresh", "activity_id" => activity.id} ) end end + test "remote poll ending notification job schedules refresh" do + user = insert(:user, local: false) + question = insert(:question, user: user) + activity = insert(:question_activity, question: question, user: user) + + PollWorker.schedule_poll_end(activity) + + expected_job_args = %{"activity_id" => activity.id, "op" => "poll_end"} + + assert_enqueued(args: expected_job_args) + + [job] = all_enqueued(worker: PollWorker) + PollWorker.perform(job) + + refute activity.local + + assert_enqueued( + worker: PollWorker, + args: %{"op" => "refresh", "activity_id" => activity.id} + ) + end + test "poll refresh" do user = insert(:user, local: false) question = insert(:question, user: user) diff --git a/test/support/factory.ex b/test/support/factory.ex index 732ea3143c..91e5805c8e 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -510,7 +510,8 @@ def question_activity_factory(attrs \\ %{}) do %Pleroma.Activity{ data: data, actor: data["actor"], - recipients: data["to"] + recipients: data["to"], + local: user.local } |> Map.merge(attrs) end From 23f78c75738aac49a79de2834490048cde817669 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Oct 2024 14:27:11 -0400 Subject: [PATCH 093/100] Refactor password changes to go through Pleroma.Web.Auth so they can be supported by the different auth backends --- lib/pleroma/web/auth/authenticator.ex | 5 ++++ lib/pleroma/web/auth/pleroma_authenticator.ex | 20 +++++++++++++ lib/pleroma/web/auth/wrapper_authenticator.ex | 4 +++ .../controllers/util_controller.ex | 29 ++++++++++--------- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index 01bf1575c4..95be892cd6 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -10,4 +10,9 @@ defmodule Pleroma.Web.Auth.Authenticator do @callback handle_error(Plug.Conn.t(), any()) :: any() @callback auth_template() :: String.t() | nil @callback oauth_consumer_template() :: String.t() | nil + + @callback change_password(Pleroma.User.t(), String.t(), String.t(), String.t()) :: + {:ok, Pleroma.User.t()} | {:error, term()} + + @optional_callbacks change_password: 4 end diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 09a58eb66b..0da3f19fce 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do alias Pleroma.Registration alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.CommonAPI alias Pleroma.Web.Plugs.AuthenticationPlug import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1, fetch_user: 1] @@ -101,4 +102,23 @@ def handle_error(%Plug.Conn{} = _conn, error) do def auth_template, do: nil def oauth_consumer_template, do: nil + + @doc "Changes Pleroma.User password in the database" + def change_password(user, password, new_password, new_password) do + case CommonAPI.Utils.confirm_current_password(user, password) do + {:ok, user} -> + with {:ok, _user} <- + User.reset_password(user, %{ + password: new_password, + password_confirmation: new_password + }) do + {:ok, user} + end + + error -> + error + end + end + + def change_password(_, _, _, _), do: {:error, :password_confirmation} end diff --git a/lib/pleroma/web/auth/wrapper_authenticator.ex b/lib/pleroma/web/auth/wrapper_authenticator.ex index a077cfa416..97b901036c 100644 --- a/lib/pleroma/web/auth/wrapper_authenticator.ex +++ b/lib/pleroma/web/auth/wrapper_authenticator.ex @@ -39,4 +39,8 @@ def oauth_consumer_template do implementation().oauth_consumer_template() || Pleroma.Config.get([:auth, :oauth_consumer_template], "consumer.html") end + + @impl true + def change_password(user, password, new_password, new_password_confirmation), + do: implementation().change_password(user, password, new_password, new_password_confirmation) end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 6805233df8..aeafa195d3 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Healthcheck alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.Auth.WrapperAuthenticator, as: Authenticator alias Pleroma.Web.CommonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.WebFinger @@ -195,19 +196,21 @@ def change_password( %{assigns: %{user: user}, private: %{open_api_spex: %{body_params: body_params}}} = conn, _ ) do - case CommonAPI.Utils.confirm_current_password(user, body_params.password) do - {:ok, user} -> - with {:ok, _user} <- - User.reset_password(user, %{ - password: body_params.new_password, - password_confirmation: body_params.new_password_confirmation - }) do - json(conn, %{status: "success"}) - else - {:error, changeset} -> - {_, {error, _}} = Enum.at(changeset.errors, 0) - json(conn, %{error: "New password #{error}."}) - end + with {:ok, %User{}} <- + Authenticator.change_password( + user, + body_params.password, + body_params.new_password, + body_params.new_password_confirmation + ) do + json(conn, %{status: "success"}) + else + {:error, %Ecto.Changeset{} = changeset} -> + {_, {error, _}} = Enum.at(changeset.errors, 0) + json(conn, %{error: "New password #{error}."}) + + {:error, :password_confirmation} -> + json(conn, %{error: "New password does not match confirmation."}) {:error, msg} -> json(conn, %{error: msg}) From 67cc38b5ac0cb009a38de3b182f34bbcb97467da Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Oct 2024 15:39:38 -0400 Subject: [PATCH 094/100] Support password changes for LDAP auth backend --- lib/pleroma/ldap.ex | 30 +++++++++++++++++++--- lib/pleroma/web/auth/ldap_authenticator.ex | 9 +++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 46a2d0c17b..9c1263fcf9 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -83,6 +83,12 @@ def handle_call({:bind_user, name, password}, from, state) do end end + def handle_call({:change_password, name, password, new_password}, _from, state) do + result = change_password(state[:handle], name, password, new_password) + + {:reply, result, state, :hibernate} + end + @impl true def terminate(_, state) do handle = Keyword.get(state, :handle) @@ -162,17 +168,16 @@ defp connect do end defp bind_user(handle, name, password) do - uid = Config.get([:ldap, :uid], "cn") - base = Config.get([:ldap, :base]) + dn = make_dn(name) - case :eldap.simple_bind(handle, "#{uid}=#{name},#{base}", password) do + case :eldap.simple_bind(handle, dn, password) do :ok -> case fetch_user(name) do %User{} = user -> user _ -> - register_user(handle, base, uid, name) + register_user(handle, ldap_base(), ldap_uid(), name) end # eldap does not inform us of socket closure @@ -231,6 +236,14 @@ defp try_register(name, attributes) do end end + defp change_password(handle, name, password, new_password) do + dn = make_dn(name) + + with :ok <- :eldap.simple_bind(handle, dn, password) do + :eldap.modify_password(handle, dn, to_charlist(new_password), to_charlist(password)) + end + end + defp decode_certfile(file) do with {:ok, data} <- File.read(file) do data @@ -242,4 +255,13 @@ defp decode_certfile(file) do [] end end + + defp ldap_uid, do: to_charlist(Config.get([:ldap, :uid], "cn")) + defp ldap_base, do: to_charlist(Config.get([:ldap, :base])) + + defp make_dn(name) do + uid = ldap_uid() + base = ldap_base() + ~c"#{uid}=#{name},#{base}" + end end diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 7eb06183d3..9bdf8447d3 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -30,4 +30,13 @@ def get_user(%Plug.Conn{} = conn) do error end end + + def change_password(user, password, new_password, new_password) do + case GenServer.call(LDAP, {:change_password, user.nickname, password, new_password}) do + :ok -> {:ok, user} + e -> e + end + end + + def change_password(_, _, _, _), do: {:error, :password_confirmation} end From ff039f953043d2c15f1eb44f794a77865ab5a775 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Oct 2024 15:41:08 -0400 Subject: [PATCH 095/100] Add example OpenLDAP ldif to enable users to change their own passwords --- installation/openldap/pw_self_service.ldif | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 installation/openldap/pw_self_service.ldif diff --git a/installation/openldap/pw_self_service.ldif b/installation/openldap/pw_self_service.ldif new file mode 100644 index 0000000000..463dabbfb5 --- /dev/null +++ b/installation/openldap/pw_self_service.ldif @@ -0,0 +1,7 @@ +dn: olcDatabase={1}mdb,cn=config +changetype: modify +add: olcAccess +olcAccess: {1}to attrs=userPassword + by self write + by anonymous auth + by * none From 6bc70b8b2a7c6942bfda01bfcc301a198cf3238b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Oct 2024 15:45:09 -0400 Subject: [PATCH 096/100] Add change_password/3 to LDAP module --- lib/pleroma/ldap.ex | 4 ++++ lib/pleroma/web/auth/ldap_authenticator.ex | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 9c1263fcf9..2bc894bd8f 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -104,6 +104,10 @@ def bind_user(name, password) do GenServer.call(__MODULE__, {:bind_user, name, password}) end + def change_password(name, password, new_password) do + GenServer.call(__MODULE__, {:change_password, name, password, new_password}) + end + defp connect do ldap = Config.get(:ldap, []) host = Keyword.get(ldap, :host, "localhost") diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 9bdf8447d3..ec6601fb90 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -32,7 +32,7 @@ def get_user(%Plug.Conn{} = conn) do end def change_password(user, password, new_password, new_password) do - case GenServer.call(LDAP, {:change_password, user.nickname, password, new_password}) do + case LDAP.change_password(user.nickname, password, new_password) do :ok -> {:ok, user} e -> e end From 1da057e6a4e4f8f7ddeb0ba286a3b996c1ba7710 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Oct 2024 15:51:56 -0400 Subject: [PATCH 097/100] Reorganize the LDAP module --- lib/pleroma/ldap.ex | 50 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/ldap.ex b/lib/pleroma/ldap.ex index 2bc894bd8f..b591c2918a 100644 --- a/lib/pleroma/ldap.ex +++ b/lib/pleroma/ldap.ex @@ -15,6 +15,14 @@ def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) end + def bind_user(name, password) do + GenServer.call(__MODULE__, {:bind_user, name, password}) + end + + def change_password(name, password, new_password) do + GenServer.call(__MODULE__, {:change_password, name, password, new_password}) + end + @impl true def init(state) do case {Config.get(Pleroma.Web.Auth.Authenticator), Config.get([:ldap, :enabled])} do @@ -47,33 +55,16 @@ def handle_continue(:connect, _state), do: do_handle_connect() def handle_info(:connect, _state), do: do_handle_connect() def handle_info({:bind_after_reconnect, name, password, from}, state) do - result = bind_user(state[:handle], name, password) + result = do_bind_user(state[:handle], name, password) GenServer.reply(from, result) {:noreply, state} end - defp do_handle_connect do - state = - case connect() do - {:ok, handle} -> - :eldap.controlling_process(handle, self()) - Process.link(handle) - [handle: handle] - - _ -> - Logger.error("Failed to connect to LDAP. Retrying in 5000ms") - Process.send_after(self(), :connect, 5_000) - [] - end - - {:noreply, state} - end - @impl true def handle_call({:bind_user, name, password}, from, state) do - case bind_user(state[:handle], name, password) do + case do_bind_user(state[:handle], name, password) do :needs_reconnect -> Process.send(self(), {:bind_after_reconnect, name, password, from}, []) {:noreply, state, {:continue, :connect}} @@ -100,12 +91,21 @@ def terminate(_, state) do :ok end - def bind_user(name, password) do - GenServer.call(__MODULE__, {:bind_user, name, password}) - end + defp do_handle_connect do + state = + case connect() do + {:ok, handle} -> + :eldap.controlling_process(handle, self()) + Process.link(handle) + [handle: handle] - def change_password(name, password, new_password) do - GenServer.call(__MODULE__, {:change_password, name, password, new_password}) + _ -> + Logger.error("Failed to connect to LDAP. Retrying in 5000ms") + Process.send_after(self(), :connect, 5_000) + [] + end + + {:noreply, state} end defp connect do @@ -171,7 +171,7 @@ defp connect do end end - defp bind_user(handle, name, password) do + defp do_bind_user(handle, name, password) do dn = make_dn(name) case :eldap.simple_bind(handle, dn, password) do From b6a951cfb5e277aa265436674055a4d0b993c5b9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Oct 2024 16:20:38 -0400 Subject: [PATCH 098/100] LDAP password changing changelog --- changelog.d/ldap-password-change.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/ldap-password-change.add diff --git a/changelog.d/ldap-password-change.add b/changelog.d/ldap-password-change.add new file mode 100644 index 0000000000..7ca555ee47 --- /dev/null +++ b/changelog.d/ldap-password-change.add @@ -0,0 +1 @@ +LDAP now supports users changing their passwords From 447d52795222040baadeba3adf004aa39236123e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 12 Oct 2024 18:10:53 +0200 Subject: [PATCH 099/100] Update README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 302d95e817..2876c7a76f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Added features: Features not authored by me: - [Chat deletion](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3029) -- [AntiDuplicationPolicy, AntiMentionSpamPolicy](https://gitlab.com/soapbox-pub/rebased/-/merge_requests/249), [RemoteReportPolicy](https://gitlab.com/soapbox-pub/rebased/-/merge_requests/202) MRFs +- [AntiDuplicationPolicy and AntiMentionSpamPolicy MRFs](https://gitlab.com/soapbox-pub/rebased/-/merge_requests/249) - [Bubble timeline](https://akkoma.dev/AkkomaGang/akkoma/pulls/100) - [Hashtag following](https://akkoma.dev/AkkomaGang/akkoma/pulls/341) - [Ability to auto-approve followbacks](https://akkoma.dev/AkkomaGang/akkoma/pulls/674) From 33b1700552edbc7e1061eb5e8238dbea3db4cb8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 12 Oct 2024 20:30:36 +0200 Subject: [PATCH 100/100] Update readme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2876c7a76f..1074593fce 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Added features: - [Moderators can assign users to reports](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3670) - [Mastodon-compatible webhooks](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3683) - UI is restyled to match pl-fe visuals +- Improved compatibility with akkoma-fe (profiles saving is still missing) Features not authored by me: - [Chat deletion](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3029) @@ -26,7 +27,10 @@ Features not authored by me: There might be more, it's hard to keep track of it. I'm trying to keep the fork as close to upstream and I hope the list will eventually get much shorter. -It should be possible to migrate from Pleroma or Rebased to `pl` without issues. It is recommended to use `pl` with [`pl-fe`](https://github.com/mkljczk/pl-fe/tree/develop/packages/pl-fe) for full feature compatibility, but pleroma-fe and other frontends work fine. +**DISCLAIMER:** +Although `pl` *just works* for me, I cannot guarantee that it'll work well for you. There might be bugs I simply don't care about or I might decide to abandon the project one day. + +It should be possible to migrate from Pleroma or Rebased to `pl` without issues. It is recommended to use `pl` with [`pl-fe`](https://github.com/mkljczk/pl-fe/tree/develop/packages/pl-fe) for full feature compatibility, but pleroma-fe and other frontends work fine too. ---