Improve ingest throughput with moderation cache and post-ack fanout

config/config.exs

@@ -3,6 +3,7 @@ import Config
 config :postgrex, :json_library, JSON
 
 config :parrhesia,
+  moderation_cache_enabled: true,
   limits: [
     max_frame_bytes: 1_048_576,
     max_event_bytes: 262_144,

config/prod.exs

@@ -1,5 +1,8 @@
 import Config
 
-config :parrhesia, Parrhesia.Repo, pool_size: 32
+config :parrhesia, Parrhesia.Repo,
+  pool_size: 32,
+  queue_target: 1_000,
+  queue_interval: 5_000
 
 # Production runtime configuration lives in config/runtime.exs.

config/runtime.exs

@@ -5,10 +5,11 @@ if config_env() == :prod do
     System.get_env("DATABASE_URL") ||
       raise "environment variable DATABASE_URL is missing. Example: ecto://USER:PASS@HOST/DATABASE"
 
-  default_pool_size =
-    :parrhesia
-    |> Application.get_env(Parrhesia.Repo, [])
-    |> Keyword.get(:pool_size, 32)
+  repo_defaults = Application.get_env(:parrhesia, Parrhesia.Repo, [])
+
+  default_pool_size = Keyword.get(repo_defaults, :pool_size, 32)
+  default_queue_target = Keyword.get(repo_defaults, :queue_target, 1_000)
+  default_queue_interval = Keyword.get(repo_defaults, :queue_interval, 5_000)
 
   pool_size =
     case System.get_env("POOL_SIZE") do
@@ -16,9 +17,23 @@ if config_env() == :prod do
       value -> String.to_integer(value)
     end
 
+  queue_target =
+    case System.get_env("DB_QUEUE_TARGET_MS") do
+      nil -> default_queue_target
+      value -> String.to_integer(value)
+    end
+
+  queue_interval =
+    case System.get_env("DB_QUEUE_INTERVAL_MS") do
+      nil -> default_queue_interval
+      value -> String.to_integer(value)
+    end
+
   config :parrhesia, Parrhesia.Repo,
     url: database_url,
-    pool_size: pool_size
+    pool_size: pool_size,
+    queue_target: queue_target,
+    queue_interval: queue_interval
 
   config :parrhesia, Parrhesia.Web.Endpoint,
     port: String.to_integer(System.get_env("PORT") || "4000")

config/test.exs

@@ -12,7 +12,9 @@ config :parrhesia, Parrhesia.Web.Endpoint,
   port: test_endpoint_port,
   ip: {127, 0, 0, 1}
 
-config :parrhesia, enable_expiration_worker: false
+config :parrhesia,
+  enable_expiration_worker: false,
+  moderation_cache_enabled: false
 
 pg_host = System.get_env("PGHOST")