Separate read pool and harden fanout state handling

test/parrhesia/application_test.exs

@@ -1,6 +1,8 @@
 defmodule Parrhesia.ApplicationTest do
   use Parrhesia.IntegrationCase, async: false
 
+  alias Parrhesia.PostgresRepos
+
   test "starts the core supervision tree" do
     assert is_pid(Process.whereis(Parrhesia.Supervisor))
     assert is_pid(Process.whereis(Parrhesia.Telemetry))
@@ -25,6 +27,7 @@ defmodule Parrhesia.ApplicationTest do
     assert is_pid(Process.whereis(Parrhesia.Auth.Nip98ReplayCache))
     assert is_pid(Process.whereis(Parrhesia.API.Identity.Manager))
     assert is_pid(Process.whereis(Parrhesia.API.Sync.Manager))
+    assert Enum.all?(PostgresRepos.started_repos(), &is_pid(Process.whereis(&1)))
 
     if negentropy_enabled?() do
       assert is_pid(Process.whereis(Parrhesia.Negentropy.Sessions))

test/parrhesia/config_test.exs

@@ -18,6 +18,7 @@ defmodule Parrhesia.ConfigTest do
     assert Parrhesia.Config.get([:limits, :auth_max_age_seconds]) == 600
     assert Parrhesia.Config.get([:limits, :max_outbound_queue]) == 256
     assert Parrhesia.Config.get([:limits, :max_filter_limit]) == 500
+    assert Parrhesia.Config.get([:database, :separate_read_pool?]) == false
     assert Parrhesia.Config.get([:relay_url]) == "ws://localhost:4413/relay"
     assert Parrhesia.Config.get([:policies, :auth_required_for_writes]) == false
     assert Parrhesia.Config.get([:policies, :marmot_media_max_imeta_tags_per_event]) == 8

test/parrhesia/performance/load_soak_test.exs

@@ -5,7 +5,8 @@ defmodule Parrhesia.Performance.LoadSoakTest do
 
   @tag :performance
   test "fanout enqueue/drain stays within relaxed p95 budget" do
-    {:ok, state} = Connection.init(subscription_index: nil, max_outbound_queue: 10_000)
+    {:ok, state} =
+      Connection.init(subscription_index: nil, max_outbound_queue: 10_000, trap_exit?: false)
 
     req_payload = JSON.encode!(["REQ", "sub-load", %{"kinds" => [1]}])
 

test/parrhesia/telemetry_test.exs

@@ -12,6 +12,7 @@ defmodule Parrhesia.TelemetryTest do
     assert [:parrhesia, :connection, :outbound_queue, :depth] in metric_names
     assert [:parrhesia, :connection, :outbound_queue, :pressure] in metric_names
     assert [:parrhesia, :connection, :outbound_queue, :pressure_events, :count] in metric_names
+    assert [:parrhesia, :process, :mailbox, :depth] in metric_names
   end
 
   test "emit/3 accepts traffic-class metadata" do
@@ -22,4 +23,26 @@ defmodule Parrhesia.TelemetryTest do
                %{traffic_class: :marmot}
              )
   end
+
+  test "emit_process_mailbox_depth/2 tags process type" do
+    handler_id = "telemetry-mailbox-depth-test"
+
+    :ok =
+      :telemetry.attach(
+        handler_id,
+        [:parrhesia, :process, :mailbox],
+        fn _event_name, measurements, metadata, test_pid ->
+          send(test_pid, {:mailbox_depth, measurements, metadata})
+        end,
+        self()
+      )
+
+    on_exit(fn -> :telemetry.detach(handler_id) end)
+
+    assert :ok = Telemetry.emit_process_mailbox_depth(:connection)
+
+    assert_receive {:mailbox_depth, %{depth: depth}, %{process_type: :connection}}
+    assert is_integer(depth)
+    assert depth >= 0
+  end
 end

test/parrhesia/web/connection_test.exs

@@ -931,6 +931,30 @@ defmodule Parrhesia.Web.ConnectionTest do
     assert delivered_ids == Enum.map(events, & &1["id"])
   end
 
+  test "shutdown drains queued outbound frames before closing" do
+    state = subscribed_connection_state(outbound_drain_batch_size: 1)
+    first = live_event(String.duplicate("a", 64), 1)
+    second = live_event(String.duplicate("b", 64), 1)
+
+    assert {:ok, queued_state} =
+             Connection.handle_info(
+               {:fanout_events, [{"sub-1", first}, {"sub-1", second}]},
+               state
+             )
+
+    assert queued_state.outbound_queue_size == 2
+
+    assert {:stop, :normal, {1012, "service restart"}, frames, drained_state} =
+             Connection.handle_info({:EXIT, self(), :shutdown}, queued_state)
+
+    assert drained_state.outbound_queue_size == 0
+
+    assert Enum.map(frames, fn {:text, payload} -> JSON.decode!(payload) end) == [
+             ["EVENT", "sub-1", first],
+             ["EVENT", "sub-1", second]
+           ]
+  end
+
   test "outbound queue overflow closes connection when strategy is close" do
     state =
       subscribed_connection_state(
@@ -975,7 +999,12 @@ defmodule Parrhesia.Web.ConnectionTest do
   end
 
   defp connection_state(opts \\ []) do
-    {:ok, state} = Connection.init(Keyword.put_new(opts, :subscription_index, nil))
+    opts =
+      opts
+      |> Keyword.put_new(:subscription_index, nil)
+      |> Keyword.put_new(:trap_exit?, false)
+
+    {:ok, state} = Connection.init(opts)
     state
   end
 

test/support/runtime.ex

@@ -1,14 +1,7 @@
 defmodule Parrhesia.TestSupport.Runtime do
   @moduledoc false
 
-  @required_processes [
-    Parrhesia.Supervisor,
-    Parrhesia.Config,
-    Parrhesia.Repo,
-    Parrhesia.Subscriptions.Supervisor,
-    Parrhesia.API.Stream.Supervisor,
-    Parrhesia.Web.Endpoint
-  ]
+  alias Parrhesia.PostgresRepos
 
   def ensure_started! do
     if healthy?() do
@@ -19,7 +12,7 @@ defmodule Parrhesia.TestSupport.Runtime do
   end
 
   defp healthy? do
-    Enum.all?(@required_processes, &is_pid(Process.whereis(&1)))
+    Enum.all?(required_processes(), &is_pid(Process.whereis(&1)))
   end
 
   defp restart! do
@@ -32,4 +25,14 @@ defmodule Parrhesia.TestSupport.Runtime do
     {:ok, _apps} = Application.ensure_all_started(:parrhesia)
     :ok
   end
+
+  defp required_processes do
+    [
+      Parrhesia.Supervisor,
+      Parrhesia.Config,
+      Parrhesia.Subscriptions.Supervisor,
+      Parrhesia.API.Stream.Supervisor,
+      Parrhesia.Web.Endpoint
+    ] ++ PostgresRepos.started_repos()
+  end
 end