diff --git a/devenv.nix b/devenv.nix index a22332d..005bc23 100644 --- a/devenv.nix +++ b/devenv.nix @@ -92,7 +92,8 @@ in { nak # Nostr relay benchmark client nostr-bench - # Nostr reference server + # Nostr reference servers + nostr-rs-relay strfry ]; diff --git a/scripts/run_bench_compare.sh b/scripts/run_bench_compare.sh index d150d9e..861a1b6 100755 --- a/scripts/run_bench_compare.sh +++ b/scripts/run_bench_compare.sh @@ -12,6 +12,7 @@ usage: Runs the same nostr-bench suite against: 1) Parrhesia (temporary test relay via run_e2e_suite.sh) 2) strfry (ephemeral instance) + 3) nostr-rs-relay (ephemeral sqlite instance) Environment: PARRHESIA_BENCH_RUNS Number of comparison runs (default: 2) @@ -54,6 +55,11 @@ if ! command -v strfry >/dev/null 2>&1; then exit 1 fi +if ! command -v nostr-rs-relay >/dev/null 2>&1; then + echo "nostr-rs-relay not found in PATH. Enter devenv shell first." >&2 + exit 1 +fi + if ! command -v ss >/dev/null 2>&1; then echo "ss command not found; cannot detect strfry readiness." >&2 exit 1 @@ -93,6 +99,7 @@ resolve_strfry_version() { } STRFRY_VERSION="$(resolve_strfry_version)" +NOSTR_RS_RELAY_VERSION="$(nostr-rs-relay --version 2>/dev/null | head -n 1 | tr -d '\r')" NOSTR_BENCH_VERSION="$(nostr-bench --version 2>/dev/null | head -n 1 | tr -d '\r')" export PARRHESIA_BENCH_CONNECT_COUNT="${PARRHESIA_BENCH_CONNECT_COUNT:-200}" @@ -109,6 +116,7 @@ export PARRHESIA_BENCH_KEEPALIVE_SECONDS="${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5 WORK_DIR="$(mktemp -d)" STRFRY_PID="" +NOSTR_RS_PID="" cleanup() { if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then @@ -116,6 +124,11 @@ cleanup() { wait "$STRFRY_PID" 2>/dev/null || true fi + if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then + kill "$NOSTR_RS_PID" 2>/dev/null || true + wait "$NOSTR_RS_PID" 2>/dev/null || true + fi + if [[ "${KEEP_BENCH_LOGS:-0}" == "1" ]]; then echo "bench logs kept at: $WORK_DIR" else @@ -164,10 +177,51 @@ stop_strfry() { STRFRY_PID="" } +start_nostr_rs_relay() { + local run_index="$1" + local port="$2" + local relay_dir="$WORK_DIR/nostr_rs_relay_${run_index}" + + mkdir -p "$relay_dir/db" + cat >"$relay_dir/config.toml" <"$relay_dir/relay.log" 2>&1 & + NOSTR_RS_PID=$! + + for _ in {1..100}; do + if ss -ltn | grep -q ":${port} "; then + return 0 + fi + sleep 0.1 + done + + echo "nostr-rs-relay did not become ready on port ${port}" >&2 + tail -n 120 "$relay_dir/relay.log" >&2 || true + return 1 +} + +stop_nostr_rs_relay() { + if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then + kill "$NOSTR_RS_PID" 2>/dev/null || true + wait "$NOSTR_RS_PID" 2>/dev/null || true + fi + NOSTR_RS_PID="" +} + echo "Running ${RUNS} comparison run(s)..." echo "Versions:" echo " parrhesia ${PARRHESIA_VERSION}" echo " ${STRFRY_VERSION}" +echo " ${NOSTR_RS_RELAY_VERSION}" echo " ${NOSTR_BENCH_VERSION}" echo @@ -193,6 +247,19 @@ for run in $(seq 1 "$RUNS"); do fi stop_strfry + echo "[run ${run}/${RUNS}] nostr-rs-relay" + nostr_rs_log="$WORK_DIR/nostr_rs_relay_${run}.log" + nostr_rs_port=$((50000 + run)) + + start_nostr_rs_relay "$run" "$nostr_rs_port" + if ! NOSTR_RS_BENCH_RELAY_URL="ws://127.0.0.1:${nostr_rs_port}" ./scripts/run_nostr_bench_nostr_rs_relay.sh all >"$nostr_rs_log" 2>&1; then + echo "nostr-rs-relay benchmark failed. Log: $nostr_rs_log" >&2 + tail -n 120 "$nostr_rs_log" >&2 || true + stop_nostr_rs_relay + exit 1 + fi + stop_nostr_rs_relay + echo done @@ -271,6 +338,7 @@ function loadRuns(prefix) { const parrhesiaRuns = loadRuns("parrhesia"); const strfryRuns = loadRuns("strfry"); +const nostrRsRuns = loadRuns("nostr_rs_relay"); const summary = { parrhesia: { @@ -293,27 +361,100 @@ const summary = { reqTps: mean(strfryRuns.map((m) => m.reqTps)), reqSizeMiBS: mean(strfryRuns.map((m) => m.reqSizeMiBS)), }, + nostrRsRelay: { + connectAvgMs: mean(nostrRsRuns.map((m) => m.connectAvgMs)), + connectMaxMs: mean(nostrRsRuns.map((m) => m.connectMaxMs)), + echoTps: mean(nostrRsRuns.map((m) => m.echoTps)), + echoSizeMiBS: mean(nostrRsRuns.map((m) => m.echoSizeMiBS)), + eventTps: mean(nostrRsRuns.map((m) => m.eventTps)), + eventSizeMiBS: mean(nostrRsRuns.map((m) => m.eventSizeMiBS)), + reqTps: mean(nostrRsRuns.map((m) => m.reqTps)), + reqSizeMiBS: mean(nostrRsRuns.map((m) => m.reqSizeMiBS)), + }, }; -function ratioStrfryVsParrhesia(metric) { +function ratioVsParrhesia(serverKey, metric) { const p = summary.parrhesia[metric]; - const s = summary.strfry[metric]; - if (!Number.isFinite(p) || !Number.isFinite(s) || p === 0) return "n/a"; - return `${(s / p).toFixed(2)}x`; + const other = summary[serverKey][metric]; + if (!Number.isFinite(p) || !Number.isFinite(other) || p === 0) return "n/a"; + return `${(other / p).toFixed(2)}x`; } const rows = [ - ["connect avg latency (ms) ↓", toFixed(summary.parrhesia.connectAvgMs), toFixed(summary.strfry.connectAvgMs), ratioStrfryVsParrhesia("connectAvgMs")], - ["connect max latency (ms) ↓", toFixed(summary.parrhesia.connectMaxMs), toFixed(summary.strfry.connectMaxMs), ratioStrfryVsParrhesia("connectMaxMs")], - ["echo throughput (TPS) ↑", toFixed(summary.parrhesia.echoTps), toFixed(summary.strfry.echoTps), ratioStrfryVsParrhesia("echoTps")], - ["echo throughput (MiB/s) ↑", toFixed(summary.parrhesia.echoSizeMiBS), toFixed(summary.strfry.echoSizeMiBS), ratioStrfryVsParrhesia("echoSizeMiBS")], - ["event throughput (TPS) ↑", toFixed(summary.parrhesia.eventTps), toFixed(summary.strfry.eventTps), ratioStrfryVsParrhesia("eventTps")], - ["event throughput (MiB/s) ↑", toFixed(summary.parrhesia.eventSizeMiBS), toFixed(summary.strfry.eventSizeMiBS), ratioStrfryVsParrhesia("eventSizeMiBS")], - ["req throughput (TPS) ↑", toFixed(summary.parrhesia.reqTps), toFixed(summary.strfry.reqTps), ratioStrfryVsParrhesia("reqTps")], - ["req throughput (MiB/s) ↑", toFixed(summary.parrhesia.reqSizeMiBS), toFixed(summary.strfry.reqSizeMiBS), ratioStrfryVsParrhesia("reqSizeMiBS")], + [ + "connect avg latency (ms) ↓", + toFixed(summary.parrhesia.connectAvgMs), + toFixed(summary.strfry.connectAvgMs), + toFixed(summary.nostrRsRelay.connectAvgMs), + ratioVsParrhesia("strfry", "connectAvgMs"), + ratioVsParrhesia("nostrRsRelay", "connectAvgMs"), + ], + [ + "connect max latency (ms) ↓", + toFixed(summary.parrhesia.connectMaxMs), + toFixed(summary.strfry.connectMaxMs), + toFixed(summary.nostrRsRelay.connectMaxMs), + ratioVsParrhesia("strfry", "connectMaxMs"), + ratioVsParrhesia("nostrRsRelay", "connectMaxMs"), + ], + [ + "echo throughput (TPS) ↑", + toFixed(summary.parrhesia.echoTps), + toFixed(summary.strfry.echoTps), + toFixed(summary.nostrRsRelay.echoTps), + ratioVsParrhesia("strfry", "echoTps"), + ratioVsParrhesia("nostrRsRelay", "echoTps"), + ], + [ + "echo throughput (MiB/s) ↑", + toFixed(summary.parrhesia.echoSizeMiBS), + toFixed(summary.strfry.echoSizeMiBS), + toFixed(summary.nostrRsRelay.echoSizeMiBS), + ratioVsParrhesia("strfry", "echoSizeMiBS"), + ratioVsParrhesia("nostrRsRelay", "echoSizeMiBS"), + ], + [ + "event throughput (TPS) ↑", + toFixed(summary.parrhesia.eventTps), + toFixed(summary.strfry.eventTps), + toFixed(summary.nostrRsRelay.eventTps), + ratioVsParrhesia("strfry", "eventTps"), + ratioVsParrhesia("nostrRsRelay", "eventTps"), + ], + [ + "event throughput (MiB/s) ↑", + toFixed(summary.parrhesia.eventSizeMiBS), + toFixed(summary.strfry.eventSizeMiBS), + toFixed(summary.nostrRsRelay.eventSizeMiBS), + ratioVsParrhesia("strfry", "eventSizeMiBS"), + ratioVsParrhesia("nostrRsRelay", "eventSizeMiBS"), + ], + [ + "req throughput (TPS) ↑", + toFixed(summary.parrhesia.reqTps), + toFixed(summary.strfry.reqTps), + toFixed(summary.nostrRsRelay.reqTps), + ratioVsParrhesia("strfry", "reqTps"), + ratioVsParrhesia("nostrRsRelay", "reqTps"), + ], + [ + "req throughput (MiB/s) ↑", + toFixed(summary.parrhesia.reqSizeMiBS), + toFixed(summary.strfry.reqSizeMiBS), + toFixed(summary.nostrRsRelay.reqSizeMiBS), + ratioVsParrhesia("strfry", "reqSizeMiBS"), + ratioVsParrhesia("nostrRsRelay", "reqSizeMiBS"), + ], ]; -const headers = ["metric", "parrhesia", "strfry", "strfry/parrhesia"]; +const headers = [ + "metric", + "parrhesia", + "strfry", + "nostr-rs-relay", + "strfry/parrhesia", + "nostr-rs/parrhesia", +]; const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length))); function fmtRow(cols) { @@ -330,16 +471,18 @@ for (const row of rows) { } console.log("\nLegend: ↑ higher is better, ↓ lower is better."); -console.log("Ratio column is strfry/parrhesia (for ↓ metrics, <1.00x means strfry is faster).\n"); +console.log("Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster).\n"); console.log("Run details:"); for (let i = 0; i < runs; i += 1) { const p = parrhesiaRuns[i]; const s = strfryRuns[i]; + const n = nostrRsRuns[i]; console.log( ` run ${i + 1}: ` + `parrhesia(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)}) | ` + - `strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)})` + `strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)}) | ` + + `nostr-rs-relay(echo_tps=${toFixed(n.echoTps, 0)}, event_tps=${toFixed(n.eventTps, 0)}, req_tps=${toFixed(n.reqTps, 0)}, connect_avg_ms=${toFixed(n.connectAvgMs, 0)})` ); } NODE diff --git a/scripts/run_nostr_bench_nostr_rs_relay.sh b/scripts/run_nostr_bench_nostr_rs_relay.sh new file mode 100755 index 0000000..0994efd --- /dev/null +++ b/scripts/run_nostr_bench_nostr_rs_relay.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +usage() { + cat <<'EOF' +usage: + ./scripts/run_nostr_bench_nostr_rs_relay.sh [all] + ./scripts/run_nostr_bench_nostr_rs_relay.sh [nostr-bench options...] + +Runs nostr-bench against a running nostr-rs-relay instance. + +Examples: + ./scripts/run_nostr_bench_nostr_rs_relay.sh + NOSTR_RS_BENCH_RELAY_URL=ws://127.0.0.1:8080 ./scripts/run_nostr_bench_nostr_rs_relay.sh + ./scripts/run_nostr_bench_nostr_rs_relay.sh connect -c 500 -r 100 + +Relay target env vars: + NOSTR_RS_BENCH_RELAY_URL (default: ws://127.0.0.1:${NOSTR_RS_RELAY_PORT:-8080}) + NOSTR_RS_RELAY_PORT (default: 8080) + +Default "all" run can be tuned via env vars: + PARRHESIA_BENCH_CONNECT_COUNT (default: 200) + PARRHESIA_BENCH_CONNECT_RATE (default: 100) + PARRHESIA_BENCH_ECHO_COUNT (default: 100) + PARRHESIA_BENCH_ECHO_RATE (default: 50) + PARRHESIA_BENCH_ECHO_SIZE (default: 512) + PARRHESIA_BENCH_EVENT_COUNT (default: 100) + PARRHESIA_BENCH_EVENT_RATE (default: 50) + PARRHESIA_BENCH_REQ_COUNT (default: 100) + PARRHESIA_BENCH_REQ_RATE (default: 50) + PARRHESIA_BENCH_REQ_LIMIT (default: 10) + PARRHESIA_BENCH_KEEPALIVE_SECONDS (default: 5) +EOF +} + +MODE="${1:-all}" +if [[ $# -gt 0 ]]; then + shift +fi + +if [[ "$MODE" == "-h" || "$MODE" == "--help" ]]; then + usage + exit 0 +fi + +if ! command -v nostr-bench >/dev/null 2>&1; then + echo "nostr-bench not found in PATH. Enter devenv shell first." >&2 + exit 1 +fi + +RELAY_URL="${NOSTR_RS_BENCH_RELAY_URL:-ws://127.0.0.1:${NOSTR_RS_RELAY_PORT:-8080}}" + +if [[ "$MODE" == "all" && $# -gt 0 ]]; then + echo "extra arguments are only supported for explicit subcommands" >&2 + usage + exit 1 +fi + +run_default() { + echo "==> nostr-bench connect ${RELAY_URL}" + nostr-bench connect \ + --json \ + -c "${PARRHESIA_BENCH_CONNECT_COUNT:-200}" \ + -r "${PARRHESIA_BENCH_CONNECT_RATE:-100}" \ + -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + "${RELAY_URL}" + + echo + echo "==> nostr-bench echo ${RELAY_URL}" + nostr-bench echo \ + --json \ + -c "${PARRHESIA_BENCH_ECHO_COUNT:-100}" \ + -r "${PARRHESIA_BENCH_ECHO_RATE:-50}" \ + -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + --size "${PARRHESIA_BENCH_ECHO_SIZE:-512}" \ + "${RELAY_URL}" + + echo + echo "==> nostr-bench event ${RELAY_URL}" + nostr-bench event \ + --json \ + -c "${PARRHESIA_BENCH_EVENT_COUNT:-100}" \ + -r "${PARRHESIA_BENCH_EVENT_RATE:-50}" \ + -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + "${RELAY_URL}" + + echo + echo "==> nostr-bench req ${RELAY_URL}" + nostr-bench req \ + --json \ + -c "${PARRHESIA_BENCH_REQ_COUNT:-100}" \ + -r "${PARRHESIA_BENCH_REQ_RATE:-50}" \ + -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + --limit "${PARRHESIA_BENCH_REQ_LIMIT:-10}" \ + "${RELAY_URL}" +} + +case "$MODE" in + all) + run_default + ;; + connect | echo | event | req) + exec nostr-bench "$MODE" "$@" "${RELAY_URL}" + ;; + *) + echo "invalid mode: ${MODE}" >&2 + exit 1 + ;; +esac