Add nostr-rs-relay to multi-relay bench comparison
This commit is contained in:
@@ -12,6 +12,7 @@ usage:
|
||||
Runs the same nostr-bench suite against:
|
||||
1) Parrhesia (temporary test relay via run_e2e_suite.sh)
|
||||
2) strfry (ephemeral instance)
|
||||
3) nostr-rs-relay (ephemeral sqlite instance)
|
||||
|
||||
Environment:
|
||||
PARRHESIA_BENCH_RUNS Number of comparison runs (default: 2)
|
||||
@@ -54,6 +55,11 @@ if ! command -v strfry >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v nostr-rs-relay >/dev/null 2>&1; then
|
||||
echo "nostr-rs-relay not found in PATH. Enter devenv shell first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v ss >/dev/null 2>&1; then
|
||||
echo "ss command not found; cannot detect strfry readiness." >&2
|
||||
exit 1
|
||||
@@ -93,6 +99,7 @@ resolve_strfry_version() {
|
||||
}
|
||||
|
||||
STRFRY_VERSION="$(resolve_strfry_version)"
|
||||
NOSTR_RS_RELAY_VERSION="$(nostr-rs-relay --version 2>/dev/null | head -n 1 | tr -d '\r')"
|
||||
NOSTR_BENCH_VERSION="$(nostr-bench --version 2>/dev/null | head -n 1 | tr -d '\r')"
|
||||
|
||||
export PARRHESIA_BENCH_CONNECT_COUNT="${PARRHESIA_BENCH_CONNECT_COUNT:-200}"
|
||||
@@ -109,6 +116,7 @@ export PARRHESIA_BENCH_KEEPALIVE_SECONDS="${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5
|
||||
|
||||
WORK_DIR="$(mktemp -d)"
|
||||
STRFRY_PID=""
|
||||
NOSTR_RS_PID=""
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then
|
||||
@@ -116,6 +124,11 @@ cleanup() {
|
||||
wait "$STRFRY_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then
|
||||
kill "$NOSTR_RS_PID" 2>/dev/null || true
|
||||
wait "$NOSTR_RS_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [[ "${KEEP_BENCH_LOGS:-0}" == "1" ]]; then
|
||||
echo "bench logs kept at: $WORK_DIR"
|
||||
else
|
||||
@@ -164,10 +177,51 @@ stop_strfry() {
|
||||
STRFRY_PID=""
|
||||
}
|
||||
|
||||
start_nostr_rs_relay() {
|
||||
local run_index="$1"
|
||||
local port="$2"
|
||||
local relay_dir="$WORK_DIR/nostr_rs_relay_${run_index}"
|
||||
|
||||
mkdir -p "$relay_dir/db"
|
||||
cat >"$relay_dir/config.toml" <<EOF
|
||||
# generated by scripts/run_bench_compare.sh
|
||||
[database]
|
||||
engine = "sqlite"
|
||||
data_directory = "$relay_dir/db"
|
||||
|
||||
[network]
|
||||
address = "127.0.0.1"
|
||||
port = ${port}
|
||||
EOF
|
||||
|
||||
nostr-rs-relay --config "$relay_dir/config.toml" >"$relay_dir/relay.log" 2>&1 &
|
||||
NOSTR_RS_PID=$!
|
||||
|
||||
for _ in {1..100}; do
|
||||
if ss -ltn | grep -q ":${port} "; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
echo "nostr-rs-relay did not become ready on port ${port}" >&2
|
||||
tail -n 120 "$relay_dir/relay.log" >&2 || true
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_nostr_rs_relay() {
|
||||
if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then
|
||||
kill "$NOSTR_RS_PID" 2>/dev/null || true
|
||||
wait "$NOSTR_RS_PID" 2>/dev/null || true
|
||||
fi
|
||||
NOSTR_RS_PID=""
|
||||
}
|
||||
|
||||
echo "Running ${RUNS} comparison run(s)..."
|
||||
echo "Versions:"
|
||||
echo " parrhesia ${PARRHESIA_VERSION}"
|
||||
echo " ${STRFRY_VERSION}"
|
||||
echo " ${NOSTR_RS_RELAY_VERSION}"
|
||||
echo " ${NOSTR_BENCH_VERSION}"
|
||||
echo
|
||||
|
||||
@@ -193,6 +247,19 @@ for run in $(seq 1 "$RUNS"); do
|
||||
fi
|
||||
stop_strfry
|
||||
|
||||
echo "[run ${run}/${RUNS}] nostr-rs-relay"
|
||||
nostr_rs_log="$WORK_DIR/nostr_rs_relay_${run}.log"
|
||||
nostr_rs_port=$((50000 + run))
|
||||
|
||||
start_nostr_rs_relay "$run" "$nostr_rs_port"
|
||||
if ! NOSTR_RS_BENCH_RELAY_URL="ws://127.0.0.1:${nostr_rs_port}" ./scripts/run_nostr_bench_nostr_rs_relay.sh all >"$nostr_rs_log" 2>&1; then
|
||||
echo "nostr-rs-relay benchmark failed. Log: $nostr_rs_log" >&2
|
||||
tail -n 120 "$nostr_rs_log" >&2 || true
|
||||
stop_nostr_rs_relay
|
||||
exit 1
|
||||
fi
|
||||
stop_nostr_rs_relay
|
||||
|
||||
echo
|
||||
|
||||
done
|
||||
@@ -271,6 +338,7 @@ function loadRuns(prefix) {
|
||||
|
||||
const parrhesiaRuns = loadRuns("parrhesia");
|
||||
const strfryRuns = loadRuns("strfry");
|
||||
const nostrRsRuns = loadRuns("nostr_rs_relay");
|
||||
|
||||
const summary = {
|
||||
parrhesia: {
|
||||
@@ -293,27 +361,100 @@ const summary = {
|
||||
reqTps: mean(strfryRuns.map((m) => m.reqTps)),
|
||||
reqSizeMiBS: mean(strfryRuns.map((m) => m.reqSizeMiBS)),
|
||||
},
|
||||
nostrRsRelay: {
|
||||
connectAvgMs: mean(nostrRsRuns.map((m) => m.connectAvgMs)),
|
||||
connectMaxMs: mean(nostrRsRuns.map((m) => m.connectMaxMs)),
|
||||
echoTps: mean(nostrRsRuns.map((m) => m.echoTps)),
|
||||
echoSizeMiBS: mean(nostrRsRuns.map((m) => m.echoSizeMiBS)),
|
||||
eventTps: mean(nostrRsRuns.map((m) => m.eventTps)),
|
||||
eventSizeMiBS: mean(nostrRsRuns.map((m) => m.eventSizeMiBS)),
|
||||
reqTps: mean(nostrRsRuns.map((m) => m.reqTps)),
|
||||
reqSizeMiBS: mean(nostrRsRuns.map((m) => m.reqSizeMiBS)),
|
||||
},
|
||||
};
|
||||
|
||||
function ratioStrfryVsParrhesia(metric) {
|
||||
function ratioVsParrhesia(serverKey, metric) {
|
||||
const p = summary.parrhesia[metric];
|
||||
const s = summary.strfry[metric];
|
||||
if (!Number.isFinite(p) || !Number.isFinite(s) || p === 0) return "n/a";
|
||||
return `${(s / p).toFixed(2)}x`;
|
||||
const other = summary[serverKey][metric];
|
||||
if (!Number.isFinite(p) || !Number.isFinite(other) || p === 0) return "n/a";
|
||||
return `${(other / p).toFixed(2)}x`;
|
||||
}
|
||||
|
||||
const rows = [
|
||||
["connect avg latency (ms) ↓", toFixed(summary.parrhesia.connectAvgMs), toFixed(summary.strfry.connectAvgMs), ratioStrfryVsParrhesia("connectAvgMs")],
|
||||
["connect max latency (ms) ↓", toFixed(summary.parrhesia.connectMaxMs), toFixed(summary.strfry.connectMaxMs), ratioStrfryVsParrhesia("connectMaxMs")],
|
||||
["echo throughput (TPS) ↑", toFixed(summary.parrhesia.echoTps), toFixed(summary.strfry.echoTps), ratioStrfryVsParrhesia("echoTps")],
|
||||
["echo throughput (MiB/s) ↑", toFixed(summary.parrhesia.echoSizeMiBS), toFixed(summary.strfry.echoSizeMiBS), ratioStrfryVsParrhesia("echoSizeMiBS")],
|
||||
["event throughput (TPS) ↑", toFixed(summary.parrhesia.eventTps), toFixed(summary.strfry.eventTps), ratioStrfryVsParrhesia("eventTps")],
|
||||
["event throughput (MiB/s) ↑", toFixed(summary.parrhesia.eventSizeMiBS), toFixed(summary.strfry.eventSizeMiBS), ratioStrfryVsParrhesia("eventSizeMiBS")],
|
||||
["req throughput (TPS) ↑", toFixed(summary.parrhesia.reqTps), toFixed(summary.strfry.reqTps), ratioStrfryVsParrhesia("reqTps")],
|
||||
["req throughput (MiB/s) ↑", toFixed(summary.parrhesia.reqSizeMiBS), toFixed(summary.strfry.reqSizeMiBS), ratioStrfryVsParrhesia("reqSizeMiBS")],
|
||||
[
|
||||
"connect avg latency (ms) ↓",
|
||||
toFixed(summary.parrhesia.connectAvgMs),
|
||||
toFixed(summary.strfry.connectAvgMs),
|
||||
toFixed(summary.nostrRsRelay.connectAvgMs),
|
||||
ratioVsParrhesia("strfry", "connectAvgMs"),
|
||||
ratioVsParrhesia("nostrRsRelay", "connectAvgMs"),
|
||||
],
|
||||
[
|
||||
"connect max latency (ms) ↓",
|
||||
toFixed(summary.parrhesia.connectMaxMs),
|
||||
toFixed(summary.strfry.connectMaxMs),
|
||||
toFixed(summary.nostrRsRelay.connectMaxMs),
|
||||
ratioVsParrhesia("strfry", "connectMaxMs"),
|
||||
ratioVsParrhesia("nostrRsRelay", "connectMaxMs"),
|
||||
],
|
||||
[
|
||||
"echo throughput (TPS) ↑",
|
||||
toFixed(summary.parrhesia.echoTps),
|
||||
toFixed(summary.strfry.echoTps),
|
||||
toFixed(summary.nostrRsRelay.echoTps),
|
||||
ratioVsParrhesia("strfry", "echoTps"),
|
||||
ratioVsParrhesia("nostrRsRelay", "echoTps"),
|
||||
],
|
||||
[
|
||||
"echo throughput (MiB/s) ↑",
|
||||
toFixed(summary.parrhesia.echoSizeMiBS),
|
||||
toFixed(summary.strfry.echoSizeMiBS),
|
||||
toFixed(summary.nostrRsRelay.echoSizeMiBS),
|
||||
ratioVsParrhesia("strfry", "echoSizeMiBS"),
|
||||
ratioVsParrhesia("nostrRsRelay", "echoSizeMiBS"),
|
||||
],
|
||||
[
|
||||
"event throughput (TPS) ↑",
|
||||
toFixed(summary.parrhesia.eventTps),
|
||||
toFixed(summary.strfry.eventTps),
|
||||
toFixed(summary.nostrRsRelay.eventTps),
|
||||
ratioVsParrhesia("strfry", "eventTps"),
|
||||
ratioVsParrhesia("nostrRsRelay", "eventTps"),
|
||||
],
|
||||
[
|
||||
"event throughput (MiB/s) ↑",
|
||||
toFixed(summary.parrhesia.eventSizeMiBS),
|
||||
toFixed(summary.strfry.eventSizeMiBS),
|
||||
toFixed(summary.nostrRsRelay.eventSizeMiBS),
|
||||
ratioVsParrhesia("strfry", "eventSizeMiBS"),
|
||||
ratioVsParrhesia("nostrRsRelay", "eventSizeMiBS"),
|
||||
],
|
||||
[
|
||||
"req throughput (TPS) ↑",
|
||||
toFixed(summary.parrhesia.reqTps),
|
||||
toFixed(summary.strfry.reqTps),
|
||||
toFixed(summary.nostrRsRelay.reqTps),
|
||||
ratioVsParrhesia("strfry", "reqTps"),
|
||||
ratioVsParrhesia("nostrRsRelay", "reqTps"),
|
||||
],
|
||||
[
|
||||
"req throughput (MiB/s) ↑",
|
||||
toFixed(summary.parrhesia.reqSizeMiBS),
|
||||
toFixed(summary.strfry.reqSizeMiBS),
|
||||
toFixed(summary.nostrRsRelay.reqSizeMiBS),
|
||||
ratioVsParrhesia("strfry", "reqSizeMiBS"),
|
||||
ratioVsParrhesia("nostrRsRelay", "reqSizeMiBS"),
|
||||
],
|
||||
];
|
||||
|
||||
const headers = ["metric", "parrhesia", "strfry", "strfry/parrhesia"];
|
||||
const headers = [
|
||||
"metric",
|
||||
"parrhesia",
|
||||
"strfry",
|
||||
"nostr-rs-relay",
|
||||
"strfry/parrhesia",
|
||||
"nostr-rs/parrhesia",
|
||||
];
|
||||
const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
|
||||
|
||||
function fmtRow(cols) {
|
||||
@@ -330,16 +471,18 @@ for (const row of rows) {
|
||||
}
|
||||
|
||||
console.log("\nLegend: ↑ higher is better, ↓ lower is better.");
|
||||
console.log("Ratio column is strfry/parrhesia (for ↓ metrics, <1.00x means strfry is faster).\n");
|
||||
console.log("Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster).\n");
|
||||
|
||||
console.log("Run details:");
|
||||
for (let i = 0; i < runs; i += 1) {
|
||||
const p = parrhesiaRuns[i];
|
||||
const s = strfryRuns[i];
|
||||
const n = nostrRsRuns[i];
|
||||
console.log(
|
||||
` run ${i + 1}: ` +
|
||||
`parrhesia(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)}) | ` +
|
||||
`strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)})`
|
||||
`strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)}) | ` +
|
||||
`nostr-rs-relay(echo_tps=${toFixed(n.echoTps, 0)}, event_tps=${toFixed(n.eventTps, 0)}, req_tps=${toFixed(n.reqTps, 0)}, connect_avg_ms=${toFixed(n.connectAvgMs, 0)})`
|
||||
);
|
||||
}
|
||||
NODE
|
||||
|
||||
Reference in New Issue
Block a user