From 8b651fab1d711a18e8046271a83bcdd756a26670 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 15 Mar 2024 16:20:16 +0100 Subject: [PATCH 01/14] AttachmentValidator: Set "Link" as default type --- .../activity_pub/object_validators/attachment_validator.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 398020bff1..72975f3485 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -12,13 +12,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do @primary_key false embedded_schema do field(:id, :string) - field(:type, :string) + field(:type, :string, default: "Link") field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do - field(:type, :string) + field(:type, :string, default: "Link") field(:href, ObjectValidators.Uri) field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:width, :integer) From 48c22a67dec0b55a1bc48ffc4229352248bceb26 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 15 Mar 2024 16:21:23 +0100 Subject: [PATCH 02/14] QuestionOptionsValidator: set default AS types --- .../object_validators/question_options_validator.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex index 541945fa40..8d7f7b9fa6 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex @@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do embeds_one :replies, Replies, primary_key: false do field(:totalItems, :integer) - field(:type, :string) + field(:type, :string, default: "Collection") end - field(:type, :string) + field(:type, :string, default: "Note") end def changeset(struct, data) do From 4ad1d02d7e5f6a3deade296b2baba374898bc033 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 15 Mar 2024 16:25:02 +0100 Subject: [PATCH 03/14] changelog.d/transient-validators-defaults.change: insert --- changelog.d/transient-validators-defaults.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/transient-validators-defaults.change diff --git a/changelog.d/transient-validators-defaults.change b/changelog.d/transient-validators-defaults.change new file mode 100644 index 0000000000..9da42dca6d --- /dev/null +++ b/changelog.d/transient-validators-defaults.change @@ -0,0 +1 @@ +Set defaults values on transient objects (attachment, poll options) validators \ No newline at end of file From 7f97fbc1ae8d38c88d32739e968b4f2572c755a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 15:36:26 -0400 Subject: [PATCH 04/14] Update minimum Postgres version to 11.0; disable JIT This release is where JIT was introduced and it should be disabled. Pleroma's queries do not benefit from JIT, but it can increase latency of queries. --- .gitlab-ci.yml | 2 +- changelog.d/postgres-jit.change | 1 + config/config.exs | 2 +- docs/installation/debian_based_jp.md | 4 ++-- docs/installation/generic_dependencies.include | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/postgres-jit.change diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 09ce2efd90..79bb2fd7e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -113,7 +113,7 @@ benchmark: variables: MIX_ENV: benchmark services: - - name: postgres:9.6-alpine + - name: postgres:11.22-alpine alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: diff --git a/changelog.d/postgres-jit.change b/changelog.d/postgres-jit.change new file mode 100644 index 0000000000..38225b06b0 --- /dev/null +++ b/changelog.d/postgres-jit.change @@ -0,0 +1 @@ +Disable jit by default for PostgreSQL diff --git a/config/config.exs b/config/config.exs index d0496cef8b..32c8509be3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -799,7 +799,7 @@ config :pleroma, configurable_from_database: false config :pleroma, Pleroma.Repo, - parameters: [gin_fuzzy_search_limit: "500"], + parameters: [gin_fuzzy_search_limit: "500", jit: "off"], prepare: :unnamed config :pleroma, :connections_pool, diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 1424ad7f48..b6b5c9c078 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -12,8 +12,8 @@ Note: This article is potentially outdated because at this time we may not have ### 必要なソフトウェア -- PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) -- `postgresql-contrib` 9.6以上 (同上) +- PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) +- `postgresql-contrib` 11.0以上 (同上) - Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index aebf21e7c7..6572716ed0 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,6 +1,6 @@ ## Required dependencies -* PostgreSQL >=9.6 +* PostgreSQL >=11.0 * Elixir >=1.11.0 <1.15 * Erlang OTP >=22.2.0 (supported: <27) * git From 1413d2e517e51a71c71d6365f93ea94fabc61593 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 15:42:15 -0400 Subject: [PATCH 05/14] Remove vestiges of old Postgres support --- lib/pleroma/application.ex | 23 +---------------------- lib/pleroma/search/database_search.ex | 9 +-------- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index de668052f2..f2b2340222 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -119,28 +119,7 @@ def start(_type, _args) do max_restarts = Application.get_env(:pleroma, __MODULE__)[:max_restarts] opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts] - result = Supervisor.start_link(children, opts) - - set_postgres_server_version() - - result - end - - defp set_postgres_server_version do - version = - with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"), - {num, _} <- Float.parse(version) do - num - else - e -> - Logger.warning( - "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6" - ) - - 9.6 - end - - :persistent_term.put({Pleroma.Repo, :postgres_version}, version) + Supervisor.start_link(children, opts) end def load_custom_modules do diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index c6311e0c77..31bfc7e338 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -23,19 +23,12 @@ def search(user, search_query, options \\ []) do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - search_function = - if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do - :websearch - else - :plain - end - try do Activity |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> restrict_public(user) - |> query_with(index_type, search_query, search_function) + |> query_with(index_type, search_query, :websearch) |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) From b822a912ad1a597484fab3465e1bd90e756abdc8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 16:15:40 -0400 Subject: [PATCH 06/14] Remove test for postgres < 11 --- test/pleroma/search/database_search_test.exs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/pleroma/search/database_search_test.exs b/test/pleroma/search/database_search_test.exs index 6c47ff4254..d8dd090639 100644 --- a/test/pleroma/search/database_search_test.exs +++ b/test/pleroma/search/database_search_test.exs @@ -35,21 +35,6 @@ test "it does not find local-only posts for anonymous users" do assert [] = Search.search(nil, "wednesday") end - test "using plainto_tsquery on postgres < 11" do - old_version = :persistent_term.get({Pleroma.Repo, :postgres_version}) - :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0) - on_exit(fn -> :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version) end) - - user = insert(:user) - {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) - {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) - - # plainto doesn't understand complex queries - assert [result] = Search.search(nil, "wednesday -dudes") - - assert result.id == post.id - end - test "using websearch_to_tsquery" do user = insert(:user) {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) From 357553a64acf8edf180621b3984e596664815248 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 16:24:23 -0400 Subject: [PATCH 07/14] Remove usage of :persistent_term for Postgres version storage, fix test This test should not have been passing. The search result's activity id should not be the same id as the local post. capture_log was not being used. Removed. --- .../controllers/search_controller_test.exs | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs index b05487abe4..ad4144da44 100644 --- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs @@ -322,26 +322,20 @@ test "search", %{conn: conn} do end test "search fetches remote statuses and prefers them over other results", %{conn: conn} do - old_version = :persistent_term.get({Pleroma.Repo, :postgres_version}) - :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0) - on_exit(fn -> :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version) end) + {:ok, %{id: activity_id}} = + CommonAPI.post(insert(:user), %{ + status: "check out http://mastodon.example.org/@admin/99541947525187367" + }) - capture_log(fn -> - {:ok, %{id: activity_id}} = - CommonAPI.post(insert(:user), %{ - status: "check out http://mastodon.example.org/@admin/99541947525187367" - }) + %{"url" => result_url, "id" => result_id} = + conn + |> get("/api/v1/search?q=http://mastodon.example.org/@admin/99541947525187367") + |> json_response_and_validate_schema(200) + |> Map.get("statuses") + |> List.first() - results = - conn - |> get("/api/v1/search?q=http://mastodon.example.org/@admin/99541947525187367") - |> json_response_and_validate_schema(200) - - assert [ - %{"url" => "http://mastodon.example.org/@admin/99541947525187367"}, - %{"id" => ^activity_id} - ] = results["statuses"] - end) + refute match?(^result_id, activity_id) + assert match?(^result_url, "http://mastodon.example.org/@admin/99541947525187367") end test "search doesn't show statuses that it shouldn't", %{conn: conn} do From 923803a5332f97a7ad4eaa1d77e38b3a7e6e2367 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 10:34:37 +0400 Subject: [PATCH 08/14] Tests: Explicitly set db pool size and max cases to the same value. --- config/test.exs | 2 +- test/test_helper.exs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 9d752bdf8e..80b01932c5 100644 --- a/config/test.exs +++ b/config/test.exs @@ -49,7 +49,7 @@ hostname: System.get_env("DB_HOST") || "localhost", port: System.get_env("DB_PORT") || "5432", pool: Ecto.Adapters.SQL.Sandbox, - pool_size: 50 + pool_size: System.schedulers_online() * 2 config :pleroma, :dangerzone, override_repo_pool_size: true diff --git a/test/test_helper.exs b/test/test_helper.exs index e65f7c1d1e..4de50483c5 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,6 +4,8 @@ Code.put_compiler_option(:warnings_as_errors, true) +ExUnit.configure(max_cases: System.schedulers_online() * 2) + ExUnit.start(exclude: [:federated, :erratic]) if match?({:unix, :darwin}, :os.type()) do From 3cc8414c2ec06e221d8157ecedd55edc6e8def8a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 10:38:29 +0400 Subject: [PATCH 09/14] Add changelog --- changelog.d/test-improvements.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/test-improvements.skip diff --git a/changelog.d/test-improvements.skip b/changelog.d/test-improvements.skip new file mode 100644 index 0000000000..e69de29bb2 From 665947ab2a55f71d8e0f457c75f48213f868f5df Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 11:03:05 +0400 Subject: [PATCH 10/14] Tests: Reduced the max case number to make tests more stable. --- test/test_helper.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index 4de50483c5..a117584ae8 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,7 +4,7 @@ Code.put_compiler_option(:warnings_as_errors, true) -ExUnit.configure(max_cases: System.schedulers_online() * 2) +ExUnit.configure(max_cases: System.schedulers_online()) ExUnit.start(exclude: [:federated, :erratic]) From 9617189e963b89f4353168e20d48ff086a0c53bb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 13:51:04 +0400 Subject: [PATCH 11/14] Tests: Actually run the bookmark folder tests. --- ...der_controller_test.ex => bookmark_folder_controller_test.exs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/pleroma/web/pleroma_api/controllers/{bookmark_folder_controller_test.ex => bookmark_folder_controller_test.exs} (100%) diff --git a/test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex b/test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.exs similarity index 100% rename from test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex rename to test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.exs From afae3a94a4d91627ec204f9e1909d0400ec6497f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 13:54:35 +0400 Subject: [PATCH 12/14] CI: Move changelog check to later in the pipeline No reason to not run tests. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79bb2fd7e4..dab52e4c6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,10 +26,10 @@ cache: &global_cache_policy - _build stages: - - check-changelog - build - lint - test + - check-changelog - benchmark - deploy - release From 040a980277947ee2eb7ae8b7432a125536dc2f06 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 14:59:25 +0400 Subject: [PATCH 13/14] Add changelog --- changelog.d/fix-bookmark-folder-tests.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/fix-bookmark-folder-tests.skip diff --git a/changelog.d/fix-bookmark-folder-tests.skip b/changelog.d/fix-bookmark-folder-tests.skip new file mode 100644 index 0000000000..e69de29bb2 From 8a14fdbe479ca0ffb3419a83cf3c637b72dd54e0 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 19 Mar 2024 12:03:43 +0000 Subject: [PATCH 14/14] Update transient-validators-defaults.change --- changelog.d/transient-validators-defaults.change | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/transient-validators-defaults.change b/changelog.d/transient-validators-defaults.change index 9da42dca6d..225cf4d0c1 100644 --- a/changelog.d/transient-validators-defaults.change +++ b/changelog.d/transient-validators-defaults.change @@ -1 +1 @@ -Set defaults values on transient objects (attachment, poll options) validators \ No newline at end of file +Set default values on validators for transient objects (attachment, poll options)