489 lines
14 KiB
Bash
Executable File
489 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT_DIR"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
usage:
|
|
./scripts/run_bench_compare.sh
|
|
|
|
Runs the same nostr-bench suite against:
|
|
1) Parrhesia (temporary test relay via run_e2e_suite.sh)
|
|
2) strfry (ephemeral instance)
|
|
3) nostr-rs-relay (ephemeral sqlite instance)
|
|
|
|
Environment:
|
|
PARRHESIA_BENCH_RUNS Number of comparison runs (default: 2)
|
|
KEEP_BENCH_LOGS Keep raw logs (1 = keep, default: 0)
|
|
|
|
Benchmark knobs (forwarded to both relays, defaults shown):
|
|
PARRHESIA_BENCH_CONNECT_COUNT 200
|
|
PARRHESIA_BENCH_CONNECT_RATE 100
|
|
PARRHESIA_BENCH_ECHO_COUNT 100
|
|
PARRHESIA_BENCH_ECHO_RATE 50
|
|
PARRHESIA_BENCH_ECHO_SIZE 512
|
|
PARRHESIA_BENCH_EVENT_COUNT 100
|
|
PARRHESIA_BENCH_EVENT_RATE 50
|
|
PARRHESIA_BENCH_REQ_COUNT 100
|
|
PARRHESIA_BENCH_REQ_RATE 50
|
|
PARRHESIA_BENCH_REQ_LIMIT 10
|
|
PARRHESIA_BENCH_KEEPALIVE_SECONDS 5
|
|
|
|
Example (quick smoke):
|
|
PARRHESIA_BENCH_RUNS=1 \
|
|
PARRHESIA_BENCH_CONNECT_COUNT=20 \
|
|
PARRHESIA_BENCH_EVENT_COUNT=20 \
|
|
PARRHESIA_BENCH_REQ_COUNT=20 \
|
|
./scripts/run_bench_compare.sh
|
|
EOF
|
|
}
|
|
|
|
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
|
|
if ! command -v nostr-bench >/dev/null 2>&1; then
|
|
echo "nostr-bench not found in PATH. Enter devenv shell first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v strfry >/dev/null 2>&1; then
|
|
echo "strfry not found in PATH. Enter devenv shell first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v nostr-rs-relay >/dev/null 2>&1; then
|
|
echo "nostr-rs-relay not found in PATH. Enter devenv shell first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v ss >/dev/null 2>&1; then
|
|
echo "ss command not found; cannot detect strfry readiness." >&2
|
|
exit 1
|
|
fi
|
|
|
|
RUNS="${PARRHESIA_BENCH_RUNS:-2}"
|
|
if ! [[ "$RUNS" =~ ^[0-9]+$ ]] || [[ "$RUNS" -lt 1 ]]; then
|
|
echo "PARRHESIA_BENCH_RUNS must be a positive integer, got: $RUNS" >&2
|
|
exit 1
|
|
fi
|
|
|
|
PARRHESIA_VERSION="$(grep -E 'version:[[:space:]]+"[^"]+"' mix.exs | head -n 1 | sed -E 's/.*version:[[:space:]]+"([^"]+)".*/\1/')"
|
|
|
|
resolve_strfry_version() {
|
|
local cli_version
|
|
local nix_version
|
|
|
|
cli_version="$(strfry --version 2>/dev/null | head -n 1 | tr -d '\r')"
|
|
|
|
# some builds report "strfry no-git-commits" instead of a semver-like version
|
|
if [[ "$cli_version" =~ [0-9]+\.[0-9]+(\.[0-9]+)? ]]; then
|
|
printf '%s\n' "$cli_version"
|
|
return 0
|
|
fi
|
|
|
|
nix_version=""
|
|
if command -v nix >/dev/null 2>&1; then
|
|
nix_version="$(nix eval --raw --impure --expr 'let pkgs = import <nixpkgs> {}; in pkgs.strfry.version' 2>/dev/null || true)"
|
|
fi
|
|
|
|
if [[ -n "$nix_version" ]]; then
|
|
printf 'strfry %s (nixpkgs)\n' "$nix_version"
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "$cli_version"
|
|
}
|
|
|
|
STRFRY_VERSION="$(resolve_strfry_version)"
|
|
NOSTR_RS_RELAY_VERSION="$(nostr-rs-relay --version 2>/dev/null | head -n 1 | tr -d '\r')"
|
|
NOSTR_BENCH_VERSION="$(nostr-bench --version 2>/dev/null | head -n 1 | tr -d '\r')"
|
|
|
|
export PARRHESIA_BENCH_CONNECT_COUNT="${PARRHESIA_BENCH_CONNECT_COUNT:-200}"
|
|
export PARRHESIA_BENCH_CONNECT_RATE="${PARRHESIA_BENCH_CONNECT_RATE:-100}"
|
|
export PARRHESIA_BENCH_ECHO_COUNT="${PARRHESIA_BENCH_ECHO_COUNT:-100}"
|
|
export PARRHESIA_BENCH_ECHO_RATE="${PARRHESIA_BENCH_ECHO_RATE:-50}"
|
|
export PARRHESIA_BENCH_ECHO_SIZE="${PARRHESIA_BENCH_ECHO_SIZE:-512}"
|
|
export PARRHESIA_BENCH_EVENT_COUNT="${PARRHESIA_BENCH_EVENT_COUNT:-100}"
|
|
export PARRHESIA_BENCH_EVENT_RATE="${PARRHESIA_BENCH_EVENT_RATE:-50}"
|
|
export PARRHESIA_BENCH_REQ_COUNT="${PARRHESIA_BENCH_REQ_COUNT:-100}"
|
|
export PARRHESIA_BENCH_REQ_RATE="${PARRHESIA_BENCH_REQ_RATE:-50}"
|
|
export PARRHESIA_BENCH_REQ_LIMIT="${PARRHESIA_BENCH_REQ_LIMIT:-10}"
|
|
export PARRHESIA_BENCH_KEEPALIVE_SECONDS="${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}"
|
|
|
|
WORK_DIR="$(mktemp -d)"
|
|
STRFRY_PID=""
|
|
NOSTR_RS_PID=""
|
|
|
|
cleanup() {
|
|
if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then
|
|
kill "$STRFRY_PID" 2>/dev/null || true
|
|
wait "$STRFRY_PID" 2>/dev/null || true
|
|
fi
|
|
|
|
if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then
|
|
kill "$NOSTR_RS_PID" 2>/dev/null || true
|
|
wait "$NOSTR_RS_PID" 2>/dev/null || true
|
|
fi
|
|
|
|
if [[ "${KEEP_BENCH_LOGS:-0}" == "1" ]]; then
|
|
echo "bench logs kept at: $WORK_DIR"
|
|
else
|
|
rm -rf "$WORK_DIR"
|
|
fi
|
|
}
|
|
|
|
trap cleanup EXIT INT TERM
|
|
|
|
start_strfry() {
|
|
local run_index="$1"
|
|
local port="$2"
|
|
local strfry_dir="$WORK_DIR/strfry_${run_index}"
|
|
|
|
mkdir -p "$strfry_dir/db"
|
|
cat >"$strfry_dir/strfry.conf" <<EOF
|
|
# generated by scripts/run_bench_compare.sh
|
|
db = "$strfry_dir/db"
|
|
relay {
|
|
bind = "127.0.0.1"
|
|
port = ${port}
|
|
nofiles = 131072
|
|
}
|
|
EOF
|
|
|
|
strfry --config "$strfry_dir/strfry.conf" relay >"$strfry_dir/strfry.log" 2>&1 &
|
|
STRFRY_PID=$!
|
|
|
|
for _ in {1..100}; do
|
|
if ss -ltn | grep -q ":${port} "; then
|
|
return 0
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
|
|
echo "strfry did not become ready on port ${port}" >&2
|
|
tail -n 120 "$strfry_dir/strfry.log" >&2 || true
|
|
return 1
|
|
}
|
|
|
|
stop_strfry() {
|
|
if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then
|
|
kill "$STRFRY_PID" 2>/dev/null || true
|
|
wait "$STRFRY_PID" 2>/dev/null || true
|
|
fi
|
|
STRFRY_PID=""
|
|
}
|
|
|
|
start_nostr_rs_relay() {
|
|
local run_index="$1"
|
|
local port="$2"
|
|
local relay_dir="$WORK_DIR/nostr_rs_relay_${run_index}"
|
|
|
|
mkdir -p "$relay_dir/db"
|
|
cat >"$relay_dir/config.toml" <<EOF
|
|
# generated by scripts/run_bench_compare.sh
|
|
[database]
|
|
engine = "sqlite"
|
|
data_directory = "$relay_dir/db"
|
|
|
|
[network]
|
|
address = "127.0.0.1"
|
|
port = ${port}
|
|
EOF
|
|
|
|
nostr-rs-relay --config "$relay_dir/config.toml" >"$relay_dir/relay.log" 2>&1 &
|
|
NOSTR_RS_PID=$!
|
|
|
|
for _ in {1..100}; do
|
|
if ss -ltn | grep -q ":${port} "; then
|
|
return 0
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
|
|
echo "nostr-rs-relay did not become ready on port ${port}" >&2
|
|
tail -n 120 "$relay_dir/relay.log" >&2 || true
|
|
return 1
|
|
}
|
|
|
|
stop_nostr_rs_relay() {
|
|
if [[ -n "$NOSTR_RS_PID" ]] && kill -0 "$NOSTR_RS_PID" 2>/dev/null; then
|
|
kill "$NOSTR_RS_PID" 2>/dev/null || true
|
|
wait "$NOSTR_RS_PID" 2>/dev/null || true
|
|
fi
|
|
NOSTR_RS_PID=""
|
|
}
|
|
|
|
echo "Running ${RUNS} comparison run(s)..."
|
|
echo "Versions:"
|
|
echo " parrhesia ${PARRHESIA_VERSION}"
|
|
echo " ${STRFRY_VERSION}"
|
|
echo " ${NOSTR_RS_RELAY_VERSION}"
|
|
echo " ${NOSTR_BENCH_VERSION}"
|
|
echo
|
|
|
|
for run in $(seq 1 "$RUNS"); do
|
|
echo "[run ${run}/${RUNS}] Parrhesia"
|
|
parrhesia_log="$WORK_DIR/parrhesia_${run}.log"
|
|
if ! ./scripts/run_nostr_bench.sh all >"$parrhesia_log" 2>&1; then
|
|
echo "Parrhesia benchmark failed. Log: $parrhesia_log" >&2
|
|
tail -n 120 "$parrhesia_log" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
echo "[run ${run}/${RUNS}] strfry"
|
|
strfry_log="$WORK_DIR/strfry_${run}.log"
|
|
strfry_port=$((49000 + run))
|
|
|
|
start_strfry "$run" "$strfry_port"
|
|
if ! STRFRY_BENCH_RELAY_URL="ws://127.0.0.1:${strfry_port}" ./scripts/run_nostr_bench_strfry.sh all >"$strfry_log" 2>&1; then
|
|
echo "strfry benchmark failed. Log: $strfry_log" >&2
|
|
tail -n 120 "$strfry_log" >&2 || true
|
|
stop_strfry
|
|
exit 1
|
|
fi
|
|
stop_strfry
|
|
|
|
echo "[run ${run}/${RUNS}] nostr-rs-relay"
|
|
nostr_rs_log="$WORK_DIR/nostr_rs_relay_${run}.log"
|
|
nostr_rs_port=$((50000 + run))
|
|
|
|
start_nostr_rs_relay "$run" "$nostr_rs_port"
|
|
if ! NOSTR_RS_BENCH_RELAY_URL="ws://127.0.0.1:${nostr_rs_port}" ./scripts/run_nostr_bench_nostr_rs_relay.sh all >"$nostr_rs_log" 2>&1; then
|
|
echo "nostr-rs-relay benchmark failed. Log: $nostr_rs_log" >&2
|
|
tail -n 120 "$nostr_rs_log" >&2 || true
|
|
stop_nostr_rs_relay
|
|
exit 1
|
|
fi
|
|
stop_nostr_rs_relay
|
|
|
|
echo
|
|
|
|
done
|
|
|
|
node - "$WORK_DIR" "$RUNS" <<'NODE'
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
|
|
const workDir = process.argv[2];
|
|
const runs = Number(process.argv[3]);
|
|
|
|
function parseLog(filePath) {
|
|
const content = fs.readFileSync(filePath, "utf8");
|
|
let section = null;
|
|
const last = {};
|
|
|
|
for (const lineRaw of content.split(/\r?\n/)) {
|
|
const line = lineRaw.trim();
|
|
const match = line.match(/^==>\s+nostr-bench\s+(connect|echo|event|req)\s+/);
|
|
if (match) {
|
|
section = match[1];
|
|
continue;
|
|
}
|
|
|
|
if (!line.startsWith("{")) continue;
|
|
|
|
try {
|
|
const parsed = JSON.parse(line);
|
|
if (section) {
|
|
last[section] = parsed;
|
|
}
|
|
} catch {
|
|
// ignore non-json lines
|
|
}
|
|
}
|
|
|
|
return last;
|
|
}
|
|
|
|
function runMetrics(parsed) {
|
|
const connect = parsed.connect?.connect_stats?.success_time ?? {};
|
|
const echo = parsed.echo ?? {};
|
|
const event = parsed.event ?? {};
|
|
const req = parsed.req ?? {};
|
|
|
|
return {
|
|
connectAvgMs: Number(connect.avg ?? NaN),
|
|
connectMaxMs: Number(connect.max ?? NaN),
|
|
echoTps: Number(echo.tps ?? NaN),
|
|
echoSizeMiBS: Number(echo.size ?? NaN),
|
|
eventTps: Number(event.tps ?? NaN),
|
|
eventSizeMiBS: Number(event.size ?? NaN),
|
|
reqTps: Number(req.tps ?? NaN),
|
|
reqSizeMiBS: Number(req.size ?? NaN),
|
|
};
|
|
}
|
|
|
|
function mean(values) {
|
|
const valid = values.filter((v) => Number.isFinite(v));
|
|
if (valid.length === 0) return NaN;
|
|
return valid.reduce((a, b) => a + b, 0) / valid.length;
|
|
}
|
|
|
|
function toFixed(value, digits = 2) {
|
|
return Number.isFinite(value) ? value.toFixed(digits) : "n/a";
|
|
}
|
|
|
|
function loadRuns(prefix) {
|
|
const all = [];
|
|
for (let i = 1; i <= runs; i += 1) {
|
|
const file = path.join(workDir, `${prefix}_${i}.log`);
|
|
all.push(runMetrics(parseLog(file)));
|
|
}
|
|
return all;
|
|
}
|
|
|
|
const parrhesiaRuns = loadRuns("parrhesia");
|
|
const strfryRuns = loadRuns("strfry");
|
|
const nostrRsRuns = loadRuns("nostr_rs_relay");
|
|
|
|
const summary = {
|
|
parrhesia: {
|
|
connectAvgMs: mean(parrhesiaRuns.map((m) => m.connectAvgMs)),
|
|
connectMaxMs: mean(parrhesiaRuns.map((m) => m.connectMaxMs)),
|
|
echoTps: mean(parrhesiaRuns.map((m) => m.echoTps)),
|
|
echoSizeMiBS: mean(parrhesiaRuns.map((m) => m.echoSizeMiBS)),
|
|
eventTps: mean(parrhesiaRuns.map((m) => m.eventTps)),
|
|
eventSizeMiBS: mean(parrhesiaRuns.map((m) => m.eventSizeMiBS)),
|
|
reqTps: mean(parrhesiaRuns.map((m) => m.reqTps)),
|
|
reqSizeMiBS: mean(parrhesiaRuns.map((m) => m.reqSizeMiBS)),
|
|
},
|
|
strfry: {
|
|
connectAvgMs: mean(strfryRuns.map((m) => m.connectAvgMs)),
|
|
connectMaxMs: mean(strfryRuns.map((m) => m.connectMaxMs)),
|
|
echoTps: mean(strfryRuns.map((m) => m.echoTps)),
|
|
echoSizeMiBS: mean(strfryRuns.map((m) => m.echoSizeMiBS)),
|
|
eventTps: mean(strfryRuns.map((m) => m.eventTps)),
|
|
eventSizeMiBS: mean(strfryRuns.map((m) => m.eventSizeMiBS)),
|
|
reqTps: mean(strfryRuns.map((m) => m.reqTps)),
|
|
reqSizeMiBS: mean(strfryRuns.map((m) => m.reqSizeMiBS)),
|
|
},
|
|
nostrRsRelay: {
|
|
connectAvgMs: mean(nostrRsRuns.map((m) => m.connectAvgMs)),
|
|
connectMaxMs: mean(nostrRsRuns.map((m) => m.connectMaxMs)),
|
|
echoTps: mean(nostrRsRuns.map((m) => m.echoTps)),
|
|
echoSizeMiBS: mean(nostrRsRuns.map((m) => m.echoSizeMiBS)),
|
|
eventTps: mean(nostrRsRuns.map((m) => m.eventTps)),
|
|
eventSizeMiBS: mean(nostrRsRuns.map((m) => m.eventSizeMiBS)),
|
|
reqTps: mean(nostrRsRuns.map((m) => m.reqTps)),
|
|
reqSizeMiBS: mean(nostrRsRuns.map((m) => m.reqSizeMiBS)),
|
|
},
|
|
};
|
|
|
|
function ratioVsParrhesia(serverKey, metric) {
|
|
const p = summary.parrhesia[metric];
|
|
const other = summary[serverKey][metric];
|
|
if (!Number.isFinite(p) || !Number.isFinite(other) || p === 0) return "n/a";
|
|
return `${(other / p).toFixed(2)}x`;
|
|
}
|
|
|
|
const rows = [
|
|
[
|
|
"connect avg latency (ms) ↓",
|
|
toFixed(summary.parrhesia.connectAvgMs),
|
|
toFixed(summary.strfry.connectAvgMs),
|
|
toFixed(summary.nostrRsRelay.connectAvgMs),
|
|
ratioVsParrhesia("strfry", "connectAvgMs"),
|
|
ratioVsParrhesia("nostrRsRelay", "connectAvgMs"),
|
|
],
|
|
[
|
|
"connect max latency (ms) ↓",
|
|
toFixed(summary.parrhesia.connectMaxMs),
|
|
toFixed(summary.strfry.connectMaxMs),
|
|
toFixed(summary.nostrRsRelay.connectMaxMs),
|
|
ratioVsParrhesia("strfry", "connectMaxMs"),
|
|
ratioVsParrhesia("nostrRsRelay", "connectMaxMs"),
|
|
],
|
|
[
|
|
"echo throughput (TPS) ↑",
|
|
toFixed(summary.parrhesia.echoTps),
|
|
toFixed(summary.strfry.echoTps),
|
|
toFixed(summary.nostrRsRelay.echoTps),
|
|
ratioVsParrhesia("strfry", "echoTps"),
|
|
ratioVsParrhesia("nostrRsRelay", "echoTps"),
|
|
],
|
|
[
|
|
"echo throughput (MiB/s) ↑",
|
|
toFixed(summary.parrhesia.echoSizeMiBS),
|
|
toFixed(summary.strfry.echoSizeMiBS),
|
|
toFixed(summary.nostrRsRelay.echoSizeMiBS),
|
|
ratioVsParrhesia("strfry", "echoSizeMiBS"),
|
|
ratioVsParrhesia("nostrRsRelay", "echoSizeMiBS"),
|
|
],
|
|
[
|
|
"event throughput (TPS) ↑",
|
|
toFixed(summary.parrhesia.eventTps),
|
|
toFixed(summary.strfry.eventTps),
|
|
toFixed(summary.nostrRsRelay.eventTps),
|
|
ratioVsParrhesia("strfry", "eventTps"),
|
|
ratioVsParrhesia("nostrRsRelay", "eventTps"),
|
|
],
|
|
[
|
|
"event throughput (MiB/s) ↑",
|
|
toFixed(summary.parrhesia.eventSizeMiBS),
|
|
toFixed(summary.strfry.eventSizeMiBS),
|
|
toFixed(summary.nostrRsRelay.eventSizeMiBS),
|
|
ratioVsParrhesia("strfry", "eventSizeMiBS"),
|
|
ratioVsParrhesia("nostrRsRelay", "eventSizeMiBS"),
|
|
],
|
|
[
|
|
"req throughput (TPS) ↑",
|
|
toFixed(summary.parrhesia.reqTps),
|
|
toFixed(summary.strfry.reqTps),
|
|
toFixed(summary.nostrRsRelay.reqTps),
|
|
ratioVsParrhesia("strfry", "reqTps"),
|
|
ratioVsParrhesia("nostrRsRelay", "reqTps"),
|
|
],
|
|
[
|
|
"req throughput (MiB/s) ↑",
|
|
toFixed(summary.parrhesia.reqSizeMiBS),
|
|
toFixed(summary.strfry.reqSizeMiBS),
|
|
toFixed(summary.nostrRsRelay.reqSizeMiBS),
|
|
ratioVsParrhesia("strfry", "reqSizeMiBS"),
|
|
ratioVsParrhesia("nostrRsRelay", "reqSizeMiBS"),
|
|
],
|
|
];
|
|
|
|
const headers = [
|
|
"metric",
|
|
"parrhesia",
|
|
"strfry",
|
|
"nostr-rs-relay",
|
|
"strfry/parrhesia",
|
|
"nostr-rs/parrhesia",
|
|
];
|
|
const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
|
|
|
|
function fmtRow(cols) {
|
|
return cols
|
|
.map((c, i) => c.padEnd(widths[i]))
|
|
.join(" ");
|
|
}
|
|
|
|
console.log("=== Bench comparison (averages) ===");
|
|
console.log(fmtRow(headers));
|
|
console.log(fmtRow(widths.map((w) => "-".repeat(w))));
|
|
for (const row of rows) {
|
|
console.log(fmtRow(row));
|
|
}
|
|
|
|
console.log("\nLegend: ↑ higher is better, ↓ lower is better.");
|
|
console.log("Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster).\n");
|
|
|
|
console.log("Run details:");
|
|
for (let i = 0; i < runs; i += 1) {
|
|
const p = parrhesiaRuns[i];
|
|
const s = strfryRuns[i];
|
|
const n = nostrRsRuns[i];
|
|
console.log(
|
|
` run ${i + 1}: ` +
|
|
`parrhesia(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)}) | ` +
|
|
`strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)}) | ` +
|
|
`nostr-rs-relay(echo_tps=${toFixed(n.echoTps, 0)}, event_tps=${toFixed(n.eventTps, 0)}, req_tps=${toFixed(n.reqTps, 0)}, connect_avg_ms=${toFixed(n.connectAvgMs, 0)})`
|
|
);
|
|
}
|
|
NODE
|