tribes/parrhesia
1
0
Fork 0
Code Issues Pull Requests Actions Releases 1 Activity

fix/test: benchmark

Browse Source
This commit is contained in:
Steffen
2026-03-14 02:23:08 +01:00
parent b20dbf67df
commit d348eab69e
9 changed files with 116 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
BENCHMARK.md
View File

@@ -16,18 +16,18 @@ Versions:
=== Bench comparison (averages) ===
metric parrhesia strfry nostr-rs-relay strfry/parrhesia nostr-rs/parrhesia
-------------------------- --------- -------- -------------- ---------------- ------------------
connect avg latency (ms) ↓ 10.00 3.00 2.50 0.30x 0.25x
connect max latency (ms) ↓ 18.50 5.00 4.00 0.27x 0.22x
echo throughput (TPS) ↑ 76972.00 68204.50 158779.00 0.89x 2.06x
echo throughput (MiB/s) ↑ 42.15 38.15 86.95 0.91x 2.06x
event throughput (TPS) ↑ 1749.00 3560.00 787.50 2.04x 0.45x
event throughput (MiB/s) ↑ 1.15 2.30 0.50 2.00x 0.43x
req throughput (TPS) ↑ 2463.00 1808.00 822.00 0.73x 0.33x
req throughput (MiB/s) ↑ 13.00 11.70 2.25 0.90x 0.17x
connect avg latency (ms) ↓ 11.50 3.50 2.50 0.30x 0.22x
connect max latency (ms) ↓ 20.00 5.50 3.50 0.28x 0.17x
echo throughput (TPS) ↑ 81805.50 62033.50 162281.50 0.76x 1.98x
echo throughput (MiB/s) ↑ 44.75 34.65 88.90 0.77x 1.99x
event throughput (TPS) ↑ 1524.50 3518.00 782.50 2.31x 0.51x
event throughput (MiB/s) ↑ 1.00 2.25 0.50 2.25x 0.50x
req throughput (TPS) ↑ 2539.00 1809.00 847.00 0.71x 0.33x
req throughput (MiB/s) ↑ 12.45 11.70 2.35 0.94x 0.19x
Legend: ↑ higher is better, ↓ lower is better.
Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster).
Run details:
run 1: parrhesia(echo_tps=78336, event_tps=1796, req_tps=2493, connect_avg_ms=9) | strfry(echo_tps=70189, event_tps=3567, req_tps=1809, connect_avg_ms=3) | nostr-rs-relay(echo_tps=149317, event_tps=786, req_tps=854, connect_avg_ms=2)
run 2: parrhesia(echo_tps=75608, event_tps=1702, req_tps=2433, connect_avg_ms=11) | strfry(echo_tps=66220, event_tps=3553, req_tps=1807, connect_avg_ms=3) | nostr-rs-relay(echo_tps=168241, event_tps=789, req_tps=790, connect_avg_ms=3)
run 1: parrhesia(echo_tps=80431, event_tps=1427, req_tps=2546, connect_avg_ms=13) | strfry(echo_tps=61421, event_tps=3581, req_tps=1811, connect_avg_ms=3) | nostr-rs-relay(echo_tps=167436, event_tps=792, req_tps=897, connect_avg_ms=3)
run 2: parrhesia(echo_tps=83180, event_tps=1622, req_tps=2532, connect_avg_ms=10) | strfry(echo_tps=62646, event_tps=3455, req_tps=1807, connect_avg_ms=4) | nostr-rs-relay(echo_tps=157127, event_tps=773, req_tps=797, connect_avg_ms=2)

2
config/config.exs
View File

@@ -1,5 +1,7 @@
import Config
config :postgrex, :json_library, JSON
config :parrhesia,
limits: [
max_frame_bytes: 1_048_576,

2
config/prod.exs
View File

@@ -1,3 +1,5 @@
import Config
config :parrhesia, Parrhesia.Repo, pool_size: 32
# Production runtime configuration lives in config/runtime.exs.

13
config/runtime.exs
View File

@@ -5,9 +5,20 @@ if config_env() == :prod do
System.get_env("DATABASE_URL") ||
raise "environment variable DATABASE_URL is missing. Example: ecto://USER:PASS@HOST/DATABASE"
default_pool_size =
:parrhesia
|> Application.get_env(Parrhesia.Repo, [])
|> Keyword.get(:pool_size, 32)
pool_size =
case System.get_env("POOL_SIZE") do
nil -> default_pool_size
value -> String.to_integer(value)
end
config :parrhesia, Parrhesia.Repo,
url: database_url,
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10")
pool_size: pool_size
config :parrhesia, Parrhesia.Web.Endpoint,
port: String.to_integer(System.get_env("PORT") || "4000")

17
devenv.nix
View File

@@ -121,6 +121,23 @@ in {
services.postgres = {
enable = true;
package = pkgs.postgresql_18;
# Some tuning for the benchmark
settings = {
max_connections = 300;
shared_buffers = "1GB";
effective_cache_size = "3GB";
work_mem = "16MB";
maintenance_work_mem = "256MB";
wal_compression = "on";
checkpoint_timeout = "15min";
checkpoint_completion_target = 0.9;
min_wal_size = "1GB";
max_wal_size = "4GB";
random_page_cost = 1.1;
effective_io_concurrency = 200;
};
initialDatabases = [{name = "parrhesia_dev";} {name = "parrhesia_test";}];
initialScript = ''
CREATE ROLE dev WITH LOGIN PASSWORD 'dev' SUPERUSER;

2
mix.exs
View File

@@ -71,7 +71,7 @@ defmodule Parrhesia.MixProject do
"credo --strict --all",
"deps.unlock --unused",
"test",
"test.nak_e2e",
# "test.nak_e2e",
"test.marmot_e2e"
]
]

2
scripts/run_bench_compare.sh
View File

@@ -10,7 +10,7 @@ usage:
./scripts/run_bench_compare.sh
Runs the same nostr-bench suite against:
1) Parrhesia (temporary test relay via run_e2e_suite.sh)
1) Parrhesia (temporary prod relay via run_e2e_suite.sh)
2) strfry (ephemeral instance)
3) nostr-rs-relay (ephemeral sqlite instance)

63
scripts/run_e2e_suite.sh
View File

@@ -12,7 +12,12 @@ shift
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
export MIX_ENV=test
MIX_ENV="${PARRHESIA_E2E_MIX_ENV:-test}"
if [[ "$MIX_ENV" != "test" && "$MIX_ENV" != "prod" ]]; then
echo "PARRHESIA_E2E_MIX_ENV must be test or prod, got: $MIX_ENV" >&2
exit 1
fi
export MIX_ENV
SUITE_SLUG="$(printf '%s' "$SUITE_NAME" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9' '_')"
SUITE_UPPER="$(printf '%s' "$SUITE_SLUG" | tr '[:lower:]' '[:upper:]')"
@@ -26,12 +31,40 @@ printf -v "$PORT_ENV_VAR" '%s' "$TEST_HTTP_PORT"
export "$PORT_ENV_VAR"
if [[ -z "${PGDATABASE:-}" ]]; then
export PGDATABASE="parrhesia_${SUITE_SLUG}_test"
export PGDATABASE="parrhesia_${SUITE_SLUG}_${MIX_ENV}"
fi
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet || true
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.create --quiet
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.migrate --quiet
if [[ -z "${DATABASE_URL:-}" ]]; then
PGUSER_EFFECTIVE="${PGUSER:-${USER:-agent}}"
PGHOST_EFFECTIVE="${PGHOST:-localhost}"
PGPORT_EFFECTIVE="${PGPORT:-5432}"
# Ecto requires a URL host to be present. For unix sockets we keep a dummy
# TCP host and pass the socket directory as query option.
if [[ "$PGHOST_EFFECTIVE" == /* ]]; then
if [[ -n "${PGPASSWORD:-}" ]]; then
export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}:${PGPASSWORD}@localhost/${PGDATABASE}?socket_dir=${PGHOST_EFFECTIVE}&port=${PGPORT_EFFECTIVE}"
else
export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}@localhost/${PGDATABASE}?socket_dir=${PGHOST_EFFECTIVE}&port=${PGPORT_EFFECTIVE}"
fi
else
if [[ -n "${PGPASSWORD:-}" ]]; then
export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}:${PGPASSWORD}@${PGHOST_EFFECTIVE}:${PGPORT_EFFECTIVE}/${PGDATABASE}"
else
export DATABASE_URL="ecto://${PGUSER_EFFECTIVE}@${PGHOST_EFFECTIVE}:${PGPORT_EFFECTIVE}/${PGDATABASE}"
fi
fi
fi
if [[ "$MIX_ENV" == "test" ]]; then
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet --force || true
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.create --quiet
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.migrate --quiet
else
mix ecto.drop --quiet --force || true
mix ecto.create --quiet
mix ecto.migrate --quiet
fi
SERVER_LOG="${ROOT_DIR}/.${SUITE_SLUG}-e2e-server.log"
: > "$SERVER_LOG"
@@ -41,6 +74,14 @@ cleanup() {
kill "$SERVER_PID" 2>/dev/null || true
wait "$SERVER_PID" 2>/dev/null || true
fi
if [[ "${PARRHESIA_E2E_DROP_DB_ON_EXIT:-0}" == "1" ]]; then
if [[ "$MIX_ENV" == "test" ]]; then
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet --force || true
else
mix ecto.drop --quiet --force || true
fi
fi
}
trap cleanup EXIT INT TERM
@@ -50,7 +91,11 @@ if ss -ltn "( sport = :${TEST_HTTP_PORT} )" | tail -n +2 | grep -q .; then
exit 1
fi
PARRHESIA_TEST_HTTP_PORT="$TEST_HTTP_PORT" mix run --no-halt >"$SERVER_LOG" 2>&1 &
if [[ "$MIX_ENV" == "test" ]]; then
PARRHESIA_TEST_HTTP_PORT="$TEST_HTTP_PORT" mix run --no-halt >"$SERVER_LOG" 2>&1 &
else
PORT="$TEST_HTTP_PORT" mix run --no-halt >"$SERVER_LOG" 2>&1 &
fi
SERVER_PID=$!
READY=0
@@ -68,4 +113,8 @@ if [[ "$READY" -ne 1 ]]; then
exit 1
fi
PARRHESIA_TEST_HTTP_PORT=0 "$@"
if [[ "$MIX_ENV" == "test" ]]; then
PARRHESIA_TEST_HTTP_PORT=0 "$@"
else
"$@"
fi

16
scripts/run_nostr_bench.sh
View File

@@ -10,9 +10,16 @@ usage:
./scripts/run_nostr_bench.sh [all]
./scripts/run_nostr_bench.sh <connect|echo|event|req> [nostr-bench options...]
Runs nostr-bench against a temporary Parrhesia test server started via
Runs nostr-bench against a temporary Parrhesia prod server started via
./scripts/run_e2e_suite.sh.
Pool tuning:
POOL_SIZE optional override for prod pool size
Database lifecycle:
PGDATABASE optional override (auto-generated by default)
PARRHESIA_E2E_DROP_DB_ON_EXIT=1 drop benchmark DB on exit (default: 1)
Examples:
./scripts/run_nostr_bench.sh
./scripts/run_nostr_bench.sh connect -c 500 -r 100
@@ -54,6 +61,13 @@ if [[ "$MODE" == "all" && $# -gt 0 ]]; then
exit 1
fi
if [[ -z "${PGDATABASE:-}" ]]; then
export PGDATABASE="parrhesia_bench_prod_$(date +%s)_$RANDOM"
fi
export PARRHESIA_E2E_DROP_DB_ON_EXIT="${PARRHESIA_E2E_DROP_DB_ON_EXIT:-1}"
PARRHESIA_E2E_MIX_ENV="prod" \
exec ./scripts/run_e2e_suite.sh \
bench \
bash -lc '