From d348eab69e1ae62600ddd4d245a02e746e153e67 Mon Sep 17 00:00:00 2001 From: Steffen Beyer Date: Sat, 14 Mar 2026 02:23:08 +0100 Subject: [PATCH] fix/test: benchmark --- BENCHMARK.md | 20 ++++++------ config/config.exs | 2 ++ config/prod.exs | 2 ++ config/runtime.exs | 13 +++++++- devenv.nix | 17 ++++++++++ mix.exs | 2 +- scripts/run_bench_compare.sh | 2 +- scripts/run_e2e_suite.sh | 63 ++++++++++++++++++++++++++++++++---- scripts/run_nostr_bench.sh | 16 ++++++++- 9 files changed, 116 insertions(+), 21 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index b2db7d1..4fe4590 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -16,18 +16,18 @@ Versions: === Bench comparison (averages) === metric parrhesia strfry nostr-rs-relay strfry/parrhesia nostr-rs/parrhesia -------------------------- --------- -------- -------------- ---------------- ------------------ -connect avg latency (ms) ↓ 10.00 3.00 2.50 0.30x 0.25x -connect max latency (ms) ↓ 18.50 5.00 4.00 0.27x 0.22x -echo throughput (TPS) ↑ 76972.00 68204.50 158779.00 0.89x 2.06x -echo throughput (MiB/s) ↑ 42.15 38.15 86.95 0.91x 2.06x -event throughput (TPS) ↑ 1749.00 3560.00 787.50 2.04x 0.45x -event throughput (MiB/s) ↑ 1.15 2.30 0.50 2.00x 0.43x -req throughput (TPS) ↑ 2463.00 1808.00 822.00 0.73x 0.33x -req throughput (MiB/s) ↑ 13.00 11.70 2.25 0.90x 0.17x +connect avg latency (ms) ↓ 11.50 3.50 2.50 0.30x 0.22x +connect max latency (ms) ↓ 20.00 5.50 3.50 0.28x 0.17x +echo throughput (TPS) ↑ 81805.50 62033.50 162281.50 0.76x 1.98x +echo throughput (MiB/s) ↑ 44.75 34.65 88.90 0.77x 1.99x +event throughput (TPS) ↑ 1524.50 3518.00 782.50 2.31x 0.51x +event throughput (MiB/s) ↑ 1.00 2.25 0.50 2.25x 0.50x +req throughput (TPS) ↑ 2539.00 1809.00 847.00 0.71x 0.33x +req throughput (MiB/s) ↑ 12.45 11.70 2.35 0.94x 0.19x Legend: ↑ higher is better, ↓ lower is better. Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster). Run details: - run 1: parrhesia(echo_tps=78336, event_tps=1796, req_tps=2493, connect_avg_ms=9) | strfry(echo_tps=70189, event_tps=3567, req_tps=1809, connect_avg_ms=3) | nostr-rs-relay(echo_tps=149317, event_tps=786, req_tps=854, connect_avg_ms=2) - run 2: parrhesia(echo_tps=75608, event_tps=1702, req_tps=2433, connect_avg_ms=11) | strfry(echo_tps=66220, event_tps=3553, req_tps=1807, connect_avg_ms=3) | nostr-rs-relay(echo_tps=168241, event_tps=789, req_tps=790, connect_avg_ms=3) + run 1: parrhesia(echo_tps=80431, event_tps=1427, req_tps=2546, connect_avg_ms=13) | strfry(echo_tps=61421, event_tps=3581, req_tps=1811, connect_avg_ms=3) | nostr-rs-relay(echo_tps=167436, event_tps=792, req_tps=897, connect_avg_ms=3) + run 2: parrhesia(echo_tps=83180, event_tps=1622, req_tps=2532, connect_avg_ms=10) | strfry(echo_tps=62646, event_tps=3455, req_tps=1807, connect_avg_ms=4) | nostr-rs-relay(echo_tps=157127, event_tps=773, req_tps=797, connect_avg_ms=2) diff --git a/config/config.exs b/config/config.exs index a953ef3..cdeeae6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,5 +1,7 @@ import Config +config :postgrex, :json_library, JSON + config :parrhesia, limits: [ max_frame_bytes: 1_048_576, diff --git a/config/prod.exs b/config/prod.exs index f8e5e5a..2961960 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1,3 +1,5 @@ import Config +config :parrhesia, Parrhesia.Repo, pool_size: 32 + # Production runtime configuration lives in config/runtime.exs. diff --git a/config/runtime.exs b/config/runtime.exs index 6c197c7..a8a99ba 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -5,9 +5,20 @@ if config_env() == :prod do System.get_env("DATABASE_URL") || raise "environment variable DATABASE_URL is missing. Example: ecto://USER:PASS@HOST/DATABASE" + default_pool_size = + :parrhesia + |> Application.get_env(Parrhesia.Repo, []) + |> Keyword.get(:pool_size, 32) + + pool_size = + case System.get_env("POOL_SIZE") do + nil -> default_pool_size + value -> String.to_integer(value) + end + config :parrhesia, Parrhesia.Repo, url: database_url, - pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10") + pool_size: pool_size config :parrhesia, Parrhesia.Web.Endpoint, port: String.to_integer(System.get_env("PORT") || "4000") diff --git a/devenv.nix b/devenv.nix index 005bc23..1589d52 100644 --- a/devenv.nix +++ b/devenv.nix @@ -121,6 +121,23 @@ in { services.postgres = { enable = true; package = pkgs.postgresql_18; + + # Some tuning for the benchmark + settings = { + max_connections = 300; + shared_buffers = "1GB"; + effective_cache_size = "3GB"; + work_mem = "16MB"; + maintenance_work_mem = "256MB"; + wal_compression = "on"; + checkpoint_timeout = "15min"; + checkpoint_completion_target = 0.9; + min_wal_size = "1GB"; + max_wal_size = "4GB"; + random_page_cost = 1.1; + effective_io_concurrency = 200; + }; + initialDatabases = [{name = "parrhesia_dev";} {name = "parrhesia_test";}]; initialScript = '' CREATE ROLE dev WITH LOGIN PASSWORD 'dev' SUPERUSER; diff --git a/mix.exs b/mix.exs index a8424cb..6c1164c 100644 --- a/mix.exs +++ b/mix.exs @@ -71,7 +71,7 @@ defmodule Parrhesia.MixProject do "credo --strict --all", "deps.unlock --unused", "test", - "test.nak_e2e", + # "test.nak_e2e", "test.marmot_e2e" ] ] diff --git a/scripts/run_bench_compare.sh b/scripts/run_bench_compare.sh index 861a1b6..073d90d 100755 --- a/scripts/run_bench_compare.sh +++ b/scripts/run_bench_compare.sh @@ -10,7 +10,7 @@ usage: ./scripts/run_bench_compare.sh Runs the same nostr-bench suite against: - 1) Parrhesia (temporary test relay via run_e2e_suite.sh) + 1) Parrhesia (temporary prod relay via run_e2e_suite.sh) 2) strfry (ephemeral instance) 3) nostr-rs-relay (ephemeral sqlite instance) diff --git a/scripts/run_e2e_suite.sh b/scripts/run_e2e_suite.sh index 12abee7..d78e5e2 100755 --- a/scripts/run_e2e_suite.sh +++ b/scripts/run_e2e_suite.sh @@ -12,7 +12,12 @@ shift ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" -export MIX_ENV=test +MIX_ENV="${PARRHESIA_E2E_MIX_ENV:-test}" +if [[ "$MIX_ENV" != "test" && "$MIX_ENV" != "prod" ]]; then + echo "PARRHESIA_E2E_MIX_ENV must be test or prod, got: $MIX_ENV" >&2 + exit 1 +fi +export MIX_ENV SUITE_SLUG="$(printf '%s' "$SUITE_NAME" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9' '_')" SUITE_UPPER="$(printf '%s' "$SUITE_SLUG" | tr '[:lower:]' '[:upper:]')" @@ -26,12 +31,40 @@ printf -v "$PORT_ENV_VAR" '%s' "$TEST_HTTP_PORT" export "$PORT_ENV_VAR" if [[ -z "${PGDATABASE:-}" ]]; then - export PGDATABASE="parrhesia_${SUITE_SLUG}_test" + export PGDATABASE="parrhesia_${SUITE_SLUG}_${MIX_ENV}" fi -PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet || true -PARRHESIA_TEST_HTTP_PORT=0 mix ecto.create --quiet -PARRHESIA_TEST_HTTP_PORT=0 mix ecto.migrate --quiet +if [[ -z "${DATABASE_URL:-}" ]]; then + PGUSER_EFFECTIVE="${PGUSER:-${USER:-agent}}" + PGHOST_EFFECTIVE="${PGHOST:-localhost}" + PGPORT_EFFECTIVE="${PGPORT:-5432}" + + # Ecto requires a URL host to be present. For unix sockets we keep a dummy + # TCP host and pass the socket directory as query option. + if [[ "$PGHOST_EFFECTIVE" == /* ]]; then + if [[ -n "${PGPASSWORD:-}" ]]; then + export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}:${PGPASSWORD}@localhost/${PGDATABASE}?socket_dir=${PGHOST_EFFECTIVE}&port=${PGPORT_EFFECTIVE}" + else + export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}@localhost/${PGDATABASE}?socket_dir=${PGHOST_EFFECTIVE}&port=${PGPORT_EFFECTIVE}" + fi + else + if [[ -n "${PGPASSWORD:-}" ]]; then + export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}:${PGPASSWORD}@${PGHOST_EFFECTIVE}:${PGPORT_EFFECTIVE}/${PGDATABASE}" + else + export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}@${PGHOST_EFFECTIVE}:${PGPORT_EFFECTIVE}/${PGDATABASE}" + fi + fi +fi + +if [[ "$MIX_ENV" == "test" ]]; then + PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet --force || true + PARRHESIA_TEST_HTTP_PORT=0 mix ecto.create --quiet + PARRHESIA_TEST_HTTP_PORT=0 mix ecto.migrate --quiet +else + mix ecto.drop --quiet --force || true + mix ecto.create --quiet + mix ecto.migrate --quiet +fi SERVER_LOG="${ROOT_DIR}/.${SUITE_SLUG}-e2e-server.log" : > "$SERVER_LOG" @@ -41,6 +74,14 @@ cleanup() { kill "$SERVER_PID" 2>/dev/null || true wait "$SERVER_PID" 2>/dev/null || true fi + + if [[ "${PARRHESIA_E2E_DROP_DB_ON_EXIT:-0}" == "1" ]]; then + if [[ "$MIX_ENV" == "test" ]]; then + PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet --force || true + else + mix ecto.drop --quiet --force || true + fi + fi } trap cleanup EXIT INT TERM @@ -50,7 +91,11 @@ if ss -ltn "( sport = :${TEST_HTTP_PORT} )" | tail -n +2 | grep -q .; then exit 1 fi -PARRHESIA_TEST_HTTP_PORT="$TEST_HTTP_PORT" mix run --no-halt >"$SERVER_LOG" 2>&1 & +if [[ "$MIX_ENV" == "test" ]]; then + PARRHESIA_TEST_HTTP_PORT="$TEST_HTTP_PORT" mix run --no-halt >"$SERVER_LOG" 2>&1 & +else + PORT="$TEST_HTTP_PORT" mix run --no-halt >"$SERVER_LOG" 2>&1 & +fi SERVER_PID=$! READY=0 @@ -68,4 +113,8 @@ if [[ "$READY" -ne 1 ]]; then exit 1 fi -PARRHESIA_TEST_HTTP_PORT=0 "$@" +if [[ "$MIX_ENV" == "test" ]]; then + PARRHESIA_TEST_HTTP_PORT=0 "$@" +else + "$@" +fi diff --git a/scripts/run_nostr_bench.sh b/scripts/run_nostr_bench.sh index 381f918..b04d56e 100755 --- a/scripts/run_nostr_bench.sh +++ b/scripts/run_nostr_bench.sh @@ -10,9 +10,16 @@ usage: ./scripts/run_nostr_bench.sh [all] ./scripts/run_nostr_bench.sh [nostr-bench options...] -Runs nostr-bench against a temporary Parrhesia test server started via +Runs nostr-bench against a temporary Parrhesia prod server started via ./scripts/run_e2e_suite.sh. +Pool tuning: + POOL_SIZE optional override for prod pool size + +Database lifecycle: + PGDATABASE optional override (auto-generated by default) + PARRHESIA_E2E_DROP_DB_ON_EXIT=1 drop benchmark DB on exit (default: 1) + Examples: ./scripts/run_nostr_bench.sh ./scripts/run_nostr_bench.sh connect -c 500 -r 100 @@ -54,6 +61,13 @@ if [[ "$MODE" == "all" && $# -gt 0 ]]; then exit 1 fi +if [[ -z "${PGDATABASE:-}" ]]; then + export PGDATABASE="parrhesia_bench_prod_$(date +%s)_$RANDOM" +fi + +export PARRHESIA_E2E_DROP_DB_ON_EXIT="${PARRHESIA_E2E_DROP_DB_ON_EXIT:-1}" + +PARRHESIA_E2E_MIX_ENV="prod" \ exec ./scripts/run_e2e_suite.sh \ bench \ bash -lc '