Merge remote-tracking branch 'tusooa/from/upstream-develop/tusooa/import' into backend-new

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
This commit is contained in:
marcin mikołajczak 2024-03-15 11:08:04 +01:00
commit c39fae12bc
13 changed files with 335 additions and 94 deletions

View file

@ -163,13 +163,8 @@
description: "Pleroma: An efficient and flexible fediverse server",
short_description: "",
background_image: "/images/city.jpg",
<<<<<<< HEAD
instance_thumbnail: "/instance/thumbnail.jpeg",
favicon: "/favicon.png",
=======
instance_thumbnail: "/instance/thumbnail.png",
favicon: "/favicon.svg",
>>>>>>> 0f1452566e (Change default favicon)
limit: 5_000,
description_limit: 5_000,
remote_limit: 100_000,

View file

@ -69,17 +69,18 @@ defmodule Pleroma.Constants do
]
)
const(updatable_object_types,
do: [
"Note",
"Question",
"Audio",
"Video",
"Event",
"Article",
"Page"
]
)
@status_types [
"Note",
"Question",
"Audio",
"Video",
"Event",
"Article",
"Page"
]
const(updatable_object_types, do: @status_types)
const(status_types, do: @status_types)
const(actor_types,
do: [

View file

@ -0,0 +1,92 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.Importer do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
require Pleroma.Constants
def import_object(
%{
"type" => type,
"published" => published,
"context" => context
} = object,
%User{} = user,
opts
)
when type in Pleroma.Constants.status_types() do
fixed_object =
object
|> strip_ap_id()
|> rewrite_actor(user)
|> strip_recipients(opts)
create_data =
Utils.make_create_data(
%{
actor: user,
published: published,
object: fixed_object,
to: [],
context: context
},
%{}
)
|> Utils.lazy_put_activity_defaults()
with {:ok, activity, _meta} <-
Pipeline.common_pipeline(create_data, local: true, importing: true) do
# This is to meant to be executed in the oban queue, we only care if it succeeds,
# so don't query for and put in the object here.
{:ok, activity}
else
e -> e
end
end
def import_object(_, _, _), do: {:ok, nil}
def import_activity(%{"type" => "Create", "object" => object} = _activity, %User{} = user, opts) do
import_object(object, user, opts)
end
def import_activity(_, _, _), do: {:ok, nil}
def import_one(%{"type" => type} = object, %User{} = user, opts \\ []) do
if type in Pleroma.Constants.status_types() do
import_object(object, user, opts)
else
import_activity(object, user, opts)
end
end
defp strip_ap_id(object), do: object |> Map.drop(["id"])
defp rewrite_actor(object, user) do
object
|> Map.put("actor", user.ap_id)
end
defp strip_recipients(object, opts) do
keep_unlisted = opts[:keep_unlisted] || false
orig_is_public = Visibility.is_public?(object) and not Visibility.is_local_public?(object)
unlisted_ccs =
if keep_unlisted and orig_is_public do
[Pleroma.Constants.as_public()]
else
[]
end
object
|> Map.put("to", [])
|> Map.put("cc", [object["actor"] | unlisted_ccs])
|> Map.put("bto", [])
|> Map.put("bcc", [])
end
end

View file

@ -90,7 +90,7 @@ def validate(
%{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity,
meta
) do
with {:ok, object_data} <- cast_and_apply(object),
with {:ok, object_data} <- cast_and_apply(object, meta),
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
{:ok, create_activity} <-
create_activity
@ -109,7 +109,7 @@ def validate(
with {:ok, object_data} <-
object
|> CommonFixes.maybe_add_language_from_activity(create_activity)
|> cast_and_apply_and_stringify_with_history(),
|> cast_and_apply_and_stringify_with_history(meta),
meta = Keyword.put(meta, :object_data, object_data),
{:ok, create_activity} <-
create_activity
@ -138,7 +138,7 @@ def validate(%{"type" => type} = object, meta)
do_separate_with_history(object, fn object ->
with {:ok, object} <-
object
|> validator.cast_and_validate()
|> validator.cast_and_validate(meta)
|> Ecto.Changeset.apply_action(:insert) do
object = stringify_keys(object)
@ -228,40 +228,42 @@ def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do
def validate(o, m), do: {:error, {:validator_not_set, {o, m}}}
def cast_and_apply_and_stringify_with_history(object) do
def cast_and_apply_and_stringify_with_history(object, meta) do
do_separate_with_history(object, fn object ->
with {:ok, object_data} <- cast_and_apply(object),
with {:ok, object_data} <- cast_and_apply(object, meta),
object_data <- object_data |> stringify_keys() do
{:ok, object_data}
end
end)
end
def cast_and_apply(%{"type" => "ChatMessage"} = object) do
def cast_and_apply(object, meta \\ [])
def cast_and_apply(%{"type" => "ChatMessage"} = object, _meta) do
ChatMessageValidator.cast_and_apply(object)
end
def cast_and_apply(%{"type" => "Question"} = object) do
QuestionValidator.cast_and_apply(object)
def cast_and_apply(%{"type" => "Question"} = object, meta) do
QuestionValidator.cast_and_apply(object, meta)
end
def cast_and_apply(%{"type" => "Answer"} = object) do
def cast_and_apply(%{"type" => "Answer"} = object, _meta) do
AnswerValidator.cast_and_apply(object)
end
def cast_and_apply(%{"type" => type} = object) when type in ~w[Audio Image Video] do
AudioImageVideoValidator.cast_and_apply(object)
def cast_and_apply(%{"type" => type} = object, meta) when type in ~w[Audio Image Video] do
AudioImageVideoValidator.cast_and_apply(object, meta)
end
def cast_and_apply(%{"type" => "Event"} = object) do
EventValidator.cast_and_apply(object)
def cast_and_apply(%{"type" => "Event"} = object, meta) do
EventValidator.cast_and_apply(object, meta)
end
def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note Page] do
ArticleNotePageValidator.cast_and_apply(object)
def cast_and_apply(%{"type" => type} = object, meta) when type in ~w[Article Note Page] do
ArticleNotePageValidator.cast_and_apply(object, meta)
end
def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
def cast_and_apply(o, _meta), do: {:error, {:validator_not_set, o}}
def stringify_keys(object) when is_struct(object) do
object

View file

@ -52,7 +52,7 @@ def changeset(struct, data) do
data =
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> CommonFixes.fix_object_defaults([])
struct
|> cast(data, __schema__(:fields))

View file

@ -28,21 +28,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
field(:replies, {:array, ObjectValidators.ObjectID}, default: [])
end
def cast_and_apply(data) do
def cast_and_apply(data, meta) do
data
|> cast_data()
|> cast_data(meta)
|> apply_action(:insert)
end
def cast_and_validate(data) do
def cast_and_validate(data, meta) do
data
|> cast_data()
|> validate_data()
|> cast_data(meta)
|> validate_data(meta)
end
def cast_data(data) do
def cast_data(data, meta) do
%__MODULE__{}
|> changeset(data)
|> changeset(data, meta)
end
defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data
@ -97,10 +97,10 @@ def fix_attachments(%{"attachment" => attachment} = data) when is_map(attachment
def fix_attachments(data), do: data
defp fix(data) do
defp fix(data, meta) do
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> CommonFixes.fix_object_defaults(meta)
|> fix_url()
|> fix_tag()
|> fix_replies()
@ -113,8 +113,8 @@ defp fix(data) do
|> CommonFixes.maybe_add_content_map()
end
def changeset(struct, data) do
data = fix(data)
def changeset(struct, data, meta) do
data = fix(data, meta)
struct
|> cast(data, __schema__(:fields) -- [:attachment, :tag])
@ -122,7 +122,7 @@ def changeset(struct, data) do
|> cast_embed(:tag)
end
defp validate_data(data_cng) do
defp validate_data(data_cng, _meta) do
data_cng
|> validate_inclusion(:type, ["Article", "Note", "Page"])
|> validate_required([:id, :actor, :attributedTo, :type, :context])

View file

@ -25,21 +25,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioImageVideoValidator do
end
end
def cast_and_apply(data) do
def cast_and_apply(data, meta) do
data
|> cast_data
|> cast_data(meta)
|> apply_action(:insert)
end
def cast_and_validate(data) do
def cast_and_validate(data, meta) do
data
|> cast_data()
|> validate_data()
|> cast_data(meta)
|> validate_data(meta)
end
def cast_data(data) do
def cast_data(data, meta) do
%__MODULE__{}
|> changeset(data)
|> changeset(data, meta)
end
defp find_attachment(url) do
@ -95,18 +95,18 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
defp fix_content(data), do: data
defp fix(data) do
defp fix(data, meta) do
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> CommonFixes.fix_object_defaults(meta)
|> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> fix_url()
|> fix_content()
end
def changeset(struct, data) do
data = fix(data)
def changeset(struct, data, meta) do
data = fix(data, meta)
struct
|> cast(data, __schema__(:fields) -- [:attachment, :tag])
@ -114,7 +114,7 @@ def changeset(struct, data) do
|> cast_embed(:tag)
end
defp validate_data(data_cng) do
defp validate_data(data_cng, _meta) do
data_cng
|> validate_inclusion(:type, ~w[Audio Image Video])
|> validate_required([:id, :actor, :attributedTo, :type, :context])

View file

@ -29,7 +29,15 @@ def cast_and_filter_recipients(message, field, follower_collection, field_fallba
Map.put(message, field, data)
end
def fix_object_defaults(data) do
def dont_apply_when_importing(data, func, meta) do
if meta[:importing] do
data
else
func.(data)
end
end
def fix_object_defaults(data, meta) do
data = Maps.filter_empty_values(data)
context =
@ -45,10 +53,13 @@ def fix_object_defaults(data) do
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)
|> cast_and_filter_recipients("bcc", follower_collection)
|> Transmogrifier.fix_implicit_addressing(follower_collection)
|> dont_apply_when_importing(
&Transmogrifier.fix_implicit_addressing(&1, follower_collection),
meta
)
end
def fix_activity_addressing(activity) do
def fix_activity_addressing(activity, meta \\ []) do
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"])
activity
@ -56,7 +67,10 @@ def fix_activity_addressing(activity) do
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)
|> cast_and_filter_recipients("bcc", follower_collection)
|> Transmogrifier.fix_implicit_addressing(follower_collection)
|> dont_apply_when_importing(
&Transmogrifier.fix_implicit_addressing(&1, follower_collection),
meta
)
end
def fix_actor(data) do

View file

@ -59,7 +59,7 @@ def changeset(struct, data) do
end
# CommonFixes.fix_activity_addressing adapted for Create specific behavior
defp fix_addressing(data, object) do
defp fix_addressing(data, object, meta) do
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"])
data
@ -67,7 +67,10 @@ defp fix_addressing(data, object) do
|> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"])
|> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"])
|> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"])
|> Transmogrifier.fix_implicit_addressing(follower_collection)
|> CommonFixes.dont_apply_when_importing(
&Transmogrifier.fix_implicit_addressing(&1, follower_collection),
meta
)
end
def fix(data, meta) do
@ -76,7 +79,7 @@ def fix(data, meta) do
data
|> CommonFixes.fix_actor()
|> Map.put("context", object["context"])
|> fix_addressing(object)
|> fix_addressing(object, meta)
end
defp validate_data(cng, meta) do

View file

@ -39,35 +39,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
field(:participation_request_count, :integer, default: 0)
end
def cast_and_apply(data) do
def cast_and_apply(data, meta) do
data
|> cast_data()
|> cast_data(meta)
|> apply_action(:insert)
end
def cast_and_validate(data) do
def cast_and_validate(data, meta) do
data
|> cast_data()
|> validate_data()
|> cast_data(meta)
|> validate_data(meta)
end
@spec cast_data(map()) :: map()
def cast_data(data) do
def cast_data(data, meta) do
%__MODULE__{}
|> changeset(data)
|> changeset(data, meta)
end
defp fix(data) do
defp fix(data, meta) do
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> CommonFixes.fix_object_defaults(meta)
|> Transmogrifier.fix_emoji()
|> CommonFixes.maybe_add_language()
|> CommonFixes.maybe_add_content_map()
end
def changeset(struct, data) do
data = fix(data)
def changeset(struct, data, meta) do
data = fix(data, meta)
struct
|> cast(data, __schema__(:fields) -- [:attachment, :tag, :location])
@ -76,7 +75,7 @@ def changeset(struct, data) do
|> cast_embed(:location)
end
defp validate_data(data_cng) do
defp validate_data(data_cng, _meta) do
data_cng
|> validate_inclusion(:type, ["Event"])
|> validate_inclusion(:joinMode, ~w[free restricted invite])

View file

@ -35,21 +35,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
embeds_many(:oneOf, QuestionOptionsValidator)
end
def cast_and_apply(data) do
def cast_and_apply(data, meta) do
data
|> cast_data
|> cast_data(meta)
|> apply_action(:insert)
end
def cast_and_validate(data) do
def cast_and_validate(data, meta) do
data
|> cast_data()
|> validate_data()
|> cast_data(meta)
|> validate_data(meta)
end
def cast_data(data) do
def cast_data(data, meta) do
%__MODULE__{}
|> changeset(data)
|> changeset(data, meta)
end
defp fix_closed(data) do
@ -60,17 +60,17 @@ defp fix_closed(data) do
end
end
defp fix(data) do
defp fix(data, meta) do
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> CommonFixes.fix_object_defaults(meta)
|> CommonFixes.fix_quote_url()
|> Transmogrifier.fix_emoji()
|> fix_closed()
end
def changeset(struct, data) do
data = fix(data)
def changeset(struct, data, meta) do
data = fix(data, meta)
struct
|> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment, :tag])
@ -80,7 +80,7 @@ def changeset(struct, data) do
|> cast_embed(:tag)
end
defp validate_data(data_cng) do
defp validate_data(data_cng, _meta) do
data_cng
|> validate_inclusion(:type, ["Question"])
|> validate_required([:id, :actor, :attributedTo, :type, :context])

View file

@ -0,0 +1,135 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ImporterTest do
use Pleroma.DataCase
alias Pleroma.Activity
alias Pleroma.Web.ActivityPub.Importer
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.CommonAPI
require Pleroma.Constants
import Pleroma.Factory
describe "import_one/3" do
test "it imports an activity" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{status: "mew"})
{:ok, activity_for_import} = Transmogrifier.prepare_outgoing(activity.data)
importing_user = insert(:user)
assert {:ok, imported_activity} = Importer.import_one(activity_for_import, importing_user)
imported_activity = Activity.normalize(imported_activity)
assert imported_activity.id != activity.id
assert imported_activity.actor == importing_user.ap_id
assert imported_activity.object.data["actor"] == importing_user.ap_id
assert imported_activity.object.data["content"] == activity.object.data["content"]
assert imported_activity.object.data["published"] == activity.object.data["published"]
end
test "it strips all mentions" do
user = insert(:user)
user2 = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{status: "mew @#{user2.nickname}"})
assert user2.ap_id in activity.recipients
{:ok, activity_for_import} = Transmogrifier.prepare_outgoing(activity.data)
importing_user = insert(:user)
assert {:ok, imported_activity} = Importer.import_one(activity_for_import, importing_user)
imported_activity = Activity.normalize(imported_activity)
assert [importing_user.ap_id] == imported_activity.recipients
assert [] == imported_activity.object.data["to"]
assert [importing_user.ap_id] == imported_activity.object.data["cc"]
end
test "it keeps inReplyTo and context" do
user = insert(:user)
user2 = insert(:user)
{:ok, replied_to_activity} = CommonAPI.post(user2, %{status: "mew"})
{:ok, activity} =
CommonAPI.post(user, %{
status: "mew @#{user2.nickname}",
in_reply_to_id: replied_to_activity
})
assert user2.ap_id in activity.recipients
{:ok, activity_for_import} = Transmogrifier.prepare_outgoing(activity.data)
importing_user = insert(:user)
assert {:ok, imported_activity} = Importer.import_one(activity_for_import, importing_user)
imported_activity = Activity.normalize(imported_activity)
assert [importing_user.ap_id] == imported_activity.recipients
assert imported_activity.object.data["context"] == activity.object.data["context"]
assert imported_activity.object.data["inReplyTo"] == activity.object.data["inReplyTo"]
assert imported_activity.data["context"] == activity.data["context"]
end
test "it keeps public and unlisted posts unlisted with keep_unlisted option" do
verify_with_visibility = fn visibility, yn ->
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{status: "mew", visibility: visibility})
{:ok, activity_for_import} = Transmogrifier.prepare_outgoing(activity.data)
importing_user = insert(:user)
assert {:ok, imported_activity} =
Importer.import_one(activity_for_import, importing_user, keep_unlisted: true)
imported_activity = Activity.normalize(imported_activity)
if yn do
assert [_, _] = imported_activity.recipients
else
assert [_] = imported_activity.recipients
end
assert importing_user.ap_id in imported_activity.recipients
assert Pleroma.Constants.as_public() in imported_activity.recipients == yn
assert Pleroma.Constants.as_public() in imported_activity.data["cc"] == yn
refute Pleroma.Constants.as_public() in imported_activity.data["to"]
assert Pleroma.Constants.as_public() in imported_activity.object.data["cc"] == yn
refute Pleroma.Constants.as_public() in imported_activity.object.data["to"]
end
verify_with_visibility.("public", true)
verify_with_visibility.("unlisted", true)
verify_with_visibility.("local", false)
verify_with_visibility.("private", false)
verify_with_visibility.("direct", false)
end
test "ignores on non-Create" do
importing_user = insert(:user)
assert {:ok, nil} =
Importer.import_one(%{"type" => "Announce", "object" => %{}}, importing_user)
end
test "ignores on non-status types" do
importing_user = insert(:user)
assert {:ok, nil} =
Importer.import_one(
%{"type" => "Create", "object" => %{"type" => "ChatMessage"}},
importing_user
)
assert {:ok, nil} =
Importer.import_one(
%{"type" => "Create", "object" => %{"type" => "Answer"}},
importing_user
)
end
end
end

View file

@ -30,12 +30,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
end
test "a basic note validates", %{note: note} do
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note, [])
end
test "a note from factory validates" do
note = insert(:note)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note.data)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note.data, [])
end
end
@ -90,7 +90,7 @@ test "a Note from Roadhouse validates" do
|> File.read!()
|> Jason.decode!()
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note, [])
end
test "a Note from Convergence AP Bridge validates" do
@ -112,7 +112,7 @@ test "a note with an attachment should work", _ do
|> File.read!()
|> Jason.decode!()
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note, [])
end
test "a Note without replies/first/items validates" do
@ -125,7 +125,7 @@ test "a Note without replies/first/items validates" do
|> pop_in(["replies", "first", "items"])
|> elem(1)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note, [])
end
test "Fedibird quote post" do