tribes/parrhesia
1
0
Fork 0
Code Issues Pull Requests Actions Releases 1 Activity

Add nostr-rs-relay to multi-relay bench comparison

Browse Source
This commit is contained in:
Steffen
2026-03-14 01:44:32 +01:00
parent 3b6dd0adc3
commit 6965a880d6
3 changed files with 272 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
devenv.nix
View File

@@ -92,7 +92,8 @@ in {
nak nak
# Nostr relay benchmark client # Nostr relay benchmark client
nostr-bench nostr-bench
# Nostr reference server # Nostr reference servers
nostr-rs-relay
strfry strfry
]; ];

173
scripts/run_bench_compare.sh
View File

@@ -12,6 +12,7 @@ usage:
Runs the same nostr-bench suite against: Runs the same nostr-bench suite against:
1) Parrhesia (temporary test relay via run_e2e_suite.sh) 1) Parrhesia (temporary test relay via run_e2e_suite.sh)
2) strfry (ephemeral instance) 2) strfry (ephemeral instance)
3) nostr-rs-relay (ephemeral sqlite instance)
Environment: Environment:
PARRHESIA_BENCH_RUNS Number of comparison runs (default: 2) PARRHESIA_BENCH_RUNS Number of comparison runs (default: 2)
@@ -54,6 +55,11 @@ if ! command -v strfry >/dev/null 2>&1; then
exit 1 exit 1
fi fi
if ! command -v nostr-rs-relay >/dev/null 2>&1; then
echo "nostr-rs-relay not found in PATH. Enter devenv shell first." >&2
exit 1
fi
if ! command -v ss >/dev/null 2>&1; then if ! command -v ss >/dev/null 2>&1; then
echo "ss command not found; cannot detect strfry readiness." >&2 echo "ss command not found; cannot detect strfry readiness." >&2
exit 1 exit 1
@@ -93,6 +99,7 @@ resolve_strfry_version() {
} }
STRFRY_VERSION="$(resolve_strfry_version)" STRFRY_VERSION="$(resolve_strfry_version)"
NOSTR_RS_RELAY_VERSION="$(nostr-rs-relay --version 2>/dev/null | head -n 1 | tr -d '\r')"
NOSTR_BENCH_VERSION="$(nostr-bench --version 2>/dev/null | head -n 1 | tr -d '\r')" NOSTR_BENCH_VERSION="$(nostr-bench --version 2>/dev/null | head -n 1 | tr -d '\r')"
export PARRHESIA_BENCH_CONNECT_COUNT="${PARRHESIA_BENCH_CONNECT_COUNT:-200}" export PARRHESIA_BENCH_CONNECT_COUNT="${PARRHESIA_BENCH_CONNECT_COUNT:-200}"
@@ -109,6 +116,7 @@ export PARRHESIA_BENCH_KEEPALIVE_SECONDS="${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5
WORK_DIR="$(mktemp -d)" WORK_DIR="$(mktemp -d)"
STRFRY_PID="" STRFRY_PID=""
NOSTR_RS_PID=""
cleanup() { cleanup() {
if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then
@@ -116,6 +124,11 @@ cleanup() {
wait "$STRFRY_PID" 2>/dev/null || true wait "$STRFRY_PID" 2>/dev/null || true
fi fi
if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then
kill "$NOSTR_RS_PID" 2>/dev/null || true
wait "$NOSTR_RS_PID" 2>/dev/null || true
fi
if [[ "${KEEP_BENCH_LOGS:-0}" == "1" ]]; then if [[ "${KEEP_BENCH_LOGS:-0}" == "1" ]]; then
echo "bench logs kept at: $WORK_DIR" echo "bench logs kept at: $WORK_DIR"
else else
@@ -164,10 +177,51 @@ stop_strfry() {
STRFRY_PID="" STRFRY_PID=""
} }
start_nostr_rs_relay() {
local run_index="$1"
local port="$2"
local relay_dir="$WORK_DIR/nostr_rs_relay_${run_index}"
mkdir -p "$relay_dir/db"
cat >"$relay_dir/config.toml" <<EOF
# generated by scripts/run_bench_compare.sh
[database]
engine = "sqlite"
data_directory = "$relay_dir/db"
[network]
address = "127.0.0.1"
port = ${port}
EOF
nostr-rs-relay --config "$relay_dir/config.toml" >"$relay_dir/relay.log" 2>&1 &
NOSTR_RS_PID=$!
for _ in {1..100}; do
if ss -ltn | grep -q ":${port} "; then
return 0
fi
sleep 0.1
done
echo "nostr-rs-relay did not become ready on port ${port}" >&2
tail -n 120 "$relay_dir/relay.log" >&2 || true
return 1
}
stop_nostr_rs_relay() {
if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then
kill "$NOSTR_RS_PID" 2>/dev/null || true
wait "$NOSTR_RS_PID" 2>/dev/null || true
fi
NOSTR_RS_PID=""
}
echo "Running ${RUNS} comparison run(s)..." echo "Running ${RUNS} comparison run(s)..."
echo "Versions:" echo "Versions:"
echo " parrhesia ${PARRHESIA_VERSION}" echo " parrhesia ${PARRHESIA_VERSION}"
echo " ${STRFRY_VERSION}" echo " ${STRFRY_VERSION}"
echo " ${NOSTR_RS_RELAY_VERSION}"
echo " ${NOSTR_BENCH_VERSION}" echo " ${NOSTR_BENCH_VERSION}"
echo echo
@@ -193,6 +247,19 @@ for run in $(seq 1 "$RUNS"); do
fi fi
stop_strfry stop_strfry
echo "[run ${run}/${RUNS}] nostr-rs-relay"
nostr_rs_log="$WORK_DIR/nostr_rs_relay_${run}.log"
nostr_rs_port=$((50000 + run))
start_nostr_rs_relay "$run" "$nostr_rs_port"
if ! NOSTR_RS_BENCH_RELAY_URL="ws://127.0.0.1:${nostr_rs_port}" ./scripts/run_nostr_bench_nostr_rs_relay.sh all >"$nostr_rs_log" 2>&1; then
echo "nostr-rs-relay benchmark failed. Log: $nostr_rs_log" >&2
tail -n 120 "$nostr_rs_log" >&2 || true
stop_nostr_rs_relay
exit 1
fi
stop_nostr_rs_relay
echo echo
done done
@@ -271,6 +338,7 @@ function loadRuns(prefix) {
const parrhesiaRuns = loadRuns("parrhesia"); const parrhesiaRuns = loadRuns("parrhesia");
const strfryRuns = loadRuns("strfry"); const strfryRuns = loadRuns("strfry");
const nostrRsRuns = loadRuns("nostr_rs_relay");
const summary = { const summary = {
parrhesia: { parrhesia: {
@@ -293,27 +361,100 @@ const summary = {
reqTps: mean(strfryRuns.map((m) => m.reqTps)), reqTps: mean(strfryRuns.map((m) => m.reqTps)),
reqSizeMiBS: mean(strfryRuns.map((m) => m.reqSizeMiBS)), reqSizeMiBS: mean(strfryRuns.map((m) => m.reqSizeMiBS)),
}, },
nostrRsRelay: {
connectAvgMs: mean(nostrRsRuns.map((m) => m.connectAvgMs)),
connectMaxMs: mean(nostrRsRuns.map((m) => m.connectMaxMs)),
echoTps: mean(nostrRsRuns.map((m) => m.echoTps)),
echoSizeMiBS: mean(nostrRsRuns.map((m) => m.echoSizeMiBS)),
eventTps: mean(nostrRsRuns.map((m) => m.eventTps)),
eventSizeMiBS: mean(nostrRsRuns.map((m) => m.eventSizeMiBS)),
reqTps: mean(nostrRsRuns.map((m) => m.reqTps)),
reqSizeMiBS: mean(nostrRsRuns.map((m) => m.reqSizeMiBS)),
},
}; };
function ratioStrfryVsParrhesia(metric) { function ratioVsParrhesia(serverKey, metric) {
const p = summary.parrhesia[metric]; const p = summary.parrhesia[metric];
const s = summary.strfry[metric]; const other = summary[serverKey][metric];
if (!Number.isFinite(p) || !Number.isFinite(s) || p === 0) return "n/a"; if (!Number.isFinite(p) || !Number.isFinite(other) || p === 0) return "n/a";
return `${(s / p).toFixed(2)}x`; return `${(other / p).toFixed(2)}x`;
} }
const rows = [ const rows = [
["connect avg latency (ms) ↓", toFixed(summary.parrhesia.connectAvgMs), toFixed(summary.strfry.connectAvgMs), ratioStrfryVsParrhesia("connectAvgMs")], [
["connect max latency (ms) ↓", toFixed(summary.parrhesia.connectMaxMs), toFixed(summary.strfry.connectMaxMs), ratioStrfryVsParrhesia("connectMaxMs")], "connect avg latency (ms) ↓",
["echo throughput (TPS) ↑", toFixed(summary.parrhesia.echoTps), toFixed(summary.strfry.echoTps), ratioStrfryVsParrhesia("echoTps")], toFixed(summary.parrhesia.connectAvgMs),
["echo throughput (MiB/s) ↑", toFixed(summary.parrhesia.echoSizeMiBS), toFixed(summary.strfry.echoSizeMiBS), ratioStrfryVsParrhesia("echoSizeMiBS")], toFixed(summary.strfry.connectAvgMs),
["event throughput (TPS) ↑", toFixed(summary.parrhesia.eventTps), toFixed(summary.strfry.eventTps), ratioStrfryVsParrhesia("eventTps")], toFixed(summary.nostrRsRelay.connectAvgMs),
["event throughput (MiB/s) ↑", toFixed(summary.parrhesia.eventSizeMiBS), toFixed(summary.strfry.eventSizeMiBS), ratioStrfryVsParrhesia("eventSizeMiBS")], ratioVsParrhesia("strfry", "connectAvgMs"),
["req throughput (TPS) ↑", toFixed(summary.parrhesia.reqTps), toFixed(summary.strfry.reqTps), ratioStrfryVsParrhesia("reqTps")], ratioVsParrhesia("nostrRsRelay", "connectAvgMs"),
["req throughput (MiB/s) ↑", toFixed(summary.parrhesia.reqSizeMiBS), toFixed(summary.strfry.reqSizeMiBS), ratioStrfryVsParrhesia("reqSizeMiBS")], ],
[
"connect max latency (ms) ↓",
toFixed(summary.parrhesia.connectMaxMs),
toFixed(summary.strfry.connectMaxMs),
toFixed(summary.nostrRsRelay.connectMaxMs),
ratioVsParrhesia("strfry", "connectMaxMs"),
ratioVsParrhesia("nostrRsRelay", "connectMaxMs"),
],
[
"echo throughput (TPS) ↑",
toFixed(summary.parrhesia.echoTps),
toFixed(summary.strfry.echoTps),
toFixed(summary.nostrRsRelay.echoTps),
ratioVsParrhesia("strfry", "echoTps"),
ratioVsParrhesia("nostrRsRelay", "echoTps"),
],
[
"echo throughput (MiB/s) ↑",
toFixed(summary.parrhesia.echoSizeMiBS),
toFixed(summary.strfry.echoSizeMiBS),
toFixed(summary.nostrRsRelay.echoSizeMiBS),
ratioVsParrhesia("strfry", "echoSizeMiBS"),
ratioVsParrhesia("nostrRsRelay", "echoSizeMiBS"),
],
[
"event throughput (TPS) ↑",
toFixed(summary.parrhesia.eventTps),
toFixed(summary.strfry.eventTps),
toFixed(summary.nostrRsRelay.eventTps),
ratioVsParrhesia("strfry", "eventTps"),
ratioVsParrhesia("nostrRsRelay", "eventTps"),
],
[
"event throughput (MiB/s) ↑",
toFixed(summary.parrhesia.eventSizeMiBS),
toFixed(summary.strfry.eventSizeMiBS),
toFixed(summary.nostrRsRelay.eventSizeMiBS),
ratioVsParrhesia("strfry", "eventSizeMiBS"),
ratioVsParrhesia("nostrRsRelay", "eventSizeMiBS"),
],
[
"req throughput (TPS) ↑",
toFixed(summary.parrhesia.reqTps),
toFixed(summary.strfry.reqTps),
toFixed(summary.nostrRsRelay.reqTps),
ratioVsParrhesia("strfry", "reqTps"),
ratioVsParrhesia("nostrRsRelay", "reqTps"),
],
[
"req throughput (MiB/s) ↑",
toFixed(summary.parrhesia.reqSizeMiBS),
toFixed(summary.strfry.reqSizeMiBS),
toFixed(summary.nostrRsRelay.reqSizeMiBS),
ratioVsParrhesia("strfry", "reqSizeMiBS"),
ratioVsParrhesia("nostrRsRelay", "reqSizeMiBS"),
],
]; ];
const headers = ["metric", "parrhesia", "strfry", "strfry/parrhesia"]; const headers = [
"metric",
"parrhesia",
"strfry",
"nostr-rs-relay",
"strfry/parrhesia",
"nostr-rs/parrhesia",
];
const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length))); const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
function fmtRow(cols) { function fmtRow(cols) {
@@ -330,16 +471,18 @@ for (const row of rows) {
} }
console.log("\nLegend: ↑ higher is better, ↓ lower is better."); console.log("\nLegend: ↑ higher is better, ↓ lower is better.");
console.log("Ratio column is strfry/parrhesia (for ↓ metrics, <1.00x means strfry is faster).\n"); console.log("Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster).\n");
console.log("Run details:"); console.log("Run details:");
for (let i = 0; i < runs; i += 1) { for (let i = 0; i < runs; i += 1) {
const p = parrhesiaRuns[i]; const p = parrhesiaRuns[i];
const s = strfryRuns[i]; const s = strfryRuns[i];
const n = nostrRsRuns[i];
console.log( console.log(
` run ${i + 1}: ` + ` run ${i + 1}: ` +
`parrhesia(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)}) | ` + `parrhesia(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)}) | ` +
`strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)})` `strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)}) | ` +
`nostr-rs-relay(echo_tps=${toFixed(n.echoTps, 0)}, event_tps=${toFixed(n.eventTps, 0)}, req_tps=${toFixed(n.reqTps, 0)}, connect_avg_ms=${toFixed(n.connectAvgMs, 0)})`
); );
} }
NODE NODE

112
scripts/run_nostr_bench_nostr_rs_relay.sh Executable file
View File

@@ -0,0 +1,112 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
usage() {
cat <<'EOF'
usage:
./scripts/run_nostr_bench_nostr_rs_relay.sh [all]
./scripts/run_nostr_bench_nostr_rs_relay.sh <connect|echo|event|req> [nostr-bench options...]
Runs nostr-bench against a running nostr-rs-relay instance.
Examples:
./scripts/run_nostr_bench_nostr_rs_relay.sh
NOSTR_RS_BENCH_RELAY_URL=ws://127.0.0.1:8080 ./scripts/run_nostr_bench_nostr_rs_relay.sh
./scripts/run_nostr_bench_nostr_rs_relay.sh connect -c 500 -r 100
Relay target env vars:
NOSTR_RS_BENCH_RELAY_URL (default: ws://127.0.0.1:${NOSTR_RS_RELAY_PORT:-8080})
NOSTR_RS_RELAY_PORT (default: 8080)
Default "all" run can be tuned via env vars:
PARRHESIA_BENCH_CONNECT_COUNT (default: 200)
PARRHESIA_BENCH_CONNECT_RATE (default: 100)
PARRHESIA_BENCH_ECHO_COUNT (default: 100)
PARRHESIA_BENCH_ECHO_RATE (default: 50)
PARRHESIA_BENCH_ECHO_SIZE (default: 512)
PARRHESIA_BENCH_EVENT_COUNT (default: 100)
PARRHESIA_BENCH_EVENT_RATE (default: 50)
PARRHESIA_BENCH_REQ_COUNT (default: 100)
PARRHESIA_BENCH_REQ_RATE (default: 50)
PARRHESIA_BENCH_REQ_LIMIT (default: 10)
PARRHESIA_BENCH_KEEPALIVE_SECONDS (default: 5)
EOF
}
MODE="${1:-all}"
if [[ $# -gt 0 ]]; then
shift
fi
if [[ "$MODE" == "-h" || "$MODE" == "--help" ]]; then
usage
exit 0
fi
if ! command -v nostr-bench >/dev/null 2>&1; then
echo "nostr-bench not found in PATH. Enter devenv shell first." >&2
exit 1
fi
RELAY_URL="${NOSTR_RS_BENCH_RELAY_URL:-ws://127.0.0.1:${NOSTR_RS_RELAY_PORT:-8080}}"
if [[ "$MODE" == "all" && $# -gt 0 ]]; then
echo "extra arguments are only supported for explicit subcommands" >&2
usage
exit 1
fi
run_default() {
echo "==> nostr-bench connect ${RELAY_URL}"
nostr-bench connect \
--json \
-c "${PARRHESIA_BENCH_CONNECT_COUNT:-200}" \
-r "${PARRHESIA_BENCH_CONNECT_RATE:-100}" \
-k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
"${RELAY_URL}"
echo
echo "==> nostr-bench echo ${RELAY_URL}"
nostr-bench echo \
--json \
-c "${PARRHESIA_BENCH_ECHO_COUNT:-100}" \
-r "${PARRHESIA_BENCH_ECHO_RATE:-50}" \
-k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
--size "${PARRHESIA_BENCH_ECHO_SIZE:-512}" \
"${RELAY_URL}"
echo
echo "==> nostr-bench event ${RELAY_URL}"
nostr-bench event \
--json \
-c "${PARRHESIA_BENCH_EVENT_COUNT:-100}" \
-r "${PARRHESIA_BENCH_EVENT_RATE:-50}" \
-k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
"${RELAY_URL}"
echo
echo "==> nostr-bench req ${RELAY_URL}"
nostr-bench req \
--json \
-c "${PARRHESIA_BENCH_REQ_COUNT:-100}" \
-r "${PARRHESIA_BENCH_REQ_RATE:-50}" \
-k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
--limit "${PARRHESIA_BENCH_REQ_LIMIT:-10}" \
"${RELAY_URL}"
}
case "$MODE" in
all)
run_default
;;
connect | echo | event | req)
exec nostr-bench "$MODE" "$@" "${RELAY_URL}"
;;
*)
echo "invalid mode: ${MODE}" >&2
exit 1
;;
esac