Add nostr bench tooling and mix bench comparison

2026-03-14 01:28:32 +01:00
parent 9616383e19
commit 3b6dd0adc3
6 changed files with 609 additions and 1 deletions
--- a/devenv.nix
+++ b/devenv.nix
@@ -73,6 +73,7 @@ in {
      vips.overrideAttrs (oldAttrs: {
        buildInputs = oldAttrs.buildInputs ++ [mozjpeg];
      });
+    nostr-bench = pkgs.callPackage ./nix/nostr-bench.nix {};
  in
    with pkgs; [
      just
@@ -89,6 +90,10 @@ in {
      mermaid-cli
      # Nostr CLI client
      nak
+      # Nostr relay benchmark client
+      nostr-bench
+      # Nostr reference server
+      strfry
    ];

  # https://devenv.sh/tests/
--- a/mix.exs
+++ b/mix.exs
@@ -21,7 +21,7 @@ defmodule Parrhesia.MixProject do
  end

  def cli do
-    [preferred_envs: [precommit: :test]]
+    [preferred_envs: [precommit: :test, bench: :test]]
  end

  # Run "mix help deps" to learn about dependencies.
@@ -62,6 +62,7 @@ defmodule Parrhesia.MixProject do
      test: ["ecto.create --quiet", "ecto.migrate --quiet", "test"],
      "test.nak_e2e": ["cmd ./scripts/run_nak_e2e.sh"],
      "test.marmot_e2e": ["cmd ./scripts/run_marmot_e2e.sh"],
+      bench: ["cmd ./scripts/run_bench_compare.sh"],
      # cov: ["cmd mix coveralls.lcov"],
      lint: ["format --check-formatted", "credo"],
      precommit: [
--- a/nix/nostr-bench.nix
+++ b/nix/nostr-bench.nix
@@ -0,0 +1,28 @@
+{
+  lib,
+  fetchFromGitHub,
+  rustPlatform,
+}:
+rustPlatform.buildRustPackage rec {
+  pname = "nostr-bench";
+  version = "0.4.0";
+
+  src = fetchFromGitHub {
+    owner = "rnostr";
+    repo = pname;
+    rev = "d3ab701512b7c871707b209ef3f934936e407963";
+    hash = "sha256-F2qg1veO1iNlVUKf1b/MV+vexiy4Tt+w2aikDDbp7tU=";
+  };
+
+  cargoHash = "sha256-mh9UdYhZl6JVJEeDFnY5BjfcK+PrWBBEn1Qh7/ZX17k=";
+
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Nostr relay benchmarking tool";
+    homepage = "https://github.com/rnostr/nostr-bench";
+    license = licenses.mit;
+    mainProgram = "nostr-bench";
+    platforms = platforms.unix;
+  };
+}
--- a/scripts/run_bench_compare.sh
+++ b/scripts/run_bench_compare.sh
@@ -0,0 +1,345 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+usage() {
+  cat <<'EOF'
+usage:
+  ./scripts/run_bench_compare.sh
+
+Runs the same nostr-bench suite against:
+  1) Parrhesia (temporary test relay via run_e2e_suite.sh)
+  2) strfry (ephemeral instance)
+
+Environment:
+  PARRHESIA_BENCH_RUNS               Number of comparison runs (default: 2)
+  KEEP_BENCH_LOGS                    Keep raw logs (1 = keep, default: 0)
+
+Benchmark knobs (forwarded to both relays, defaults shown):
+  PARRHESIA_BENCH_CONNECT_COUNT      200
+  PARRHESIA_BENCH_CONNECT_RATE       100
+  PARRHESIA_BENCH_ECHO_COUNT         100
+  PARRHESIA_BENCH_ECHO_RATE          50
+  PARRHESIA_BENCH_ECHO_SIZE          512
+  PARRHESIA_BENCH_EVENT_COUNT        100
+  PARRHESIA_BENCH_EVENT_RATE         50
+  PARRHESIA_BENCH_REQ_COUNT          100
+  PARRHESIA_BENCH_REQ_RATE           50
+  PARRHESIA_BENCH_REQ_LIMIT          10
+  PARRHESIA_BENCH_KEEPALIVE_SECONDS  5
+
+Example (quick smoke):
+  PARRHESIA_BENCH_RUNS=1 \
+  PARRHESIA_BENCH_CONNECT_COUNT=20 \
+  PARRHESIA_BENCH_EVENT_COUNT=20 \
+  PARRHESIA_BENCH_REQ_COUNT=20 \
+  ./scripts/run_bench_compare.sh
+EOF
+}
+
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if ! command -v nostr-bench >/dev/null 2>&1; then
+  echo "nostr-bench not found in PATH. Enter devenv shell first." >&2
+  exit 1
+fi
+
+if ! command -v strfry >/dev/null 2>&1; then
+  echo "strfry not found in PATH. Enter devenv shell first." >&2
+  exit 1
+fi
+
+if ! command -v ss >/dev/null 2>&1; then
+  echo "ss command not found; cannot detect strfry readiness." >&2
+  exit 1
+fi
+
+RUNS="${PARRHESIA_BENCH_RUNS:-2}"
+if ! [[ "$RUNS" =~ ^[0-9]+$ ]] || [[ "$RUNS" -lt 1 ]]; then
+  echo "PARRHESIA_BENCH_RUNS must be a positive integer, got: $RUNS" >&2
+  exit 1
+fi
+
+PARRHESIA_VERSION="$(grep -E 'version:[[:space:]]+"[^"]+"' mix.exs | head -n 1 | sed -E 's/.*version:[[:space:]]+"([^"]+)".*/\1/')"
+
+resolve_strfry_version() {
+  local cli_version
+  local nix_version
+
+  cli_version="$(strfry --version 2>/dev/null | head -n 1 | tr -d '\r')"
+
+  # some builds report "strfry no-git-commits" instead of a semver-like version
+  if [[ "$cli_version" =~ [0-9]+\.[0-9]+(\.[0-9]+)? ]]; then
+    printf '%s\n' "$cli_version"
+    return 0
+  fi
+
+  nix_version=""
+  if command -v nix >/dev/null 2>&1; then
+    nix_version="$(nix eval --raw --impure --expr 'let pkgs = import <nixpkgs> {}; in pkgs.strfry.version' 2>/dev/null || true)"
+  fi
+
+  if [[ -n "$nix_version" ]]; then
+    printf 'strfry %s (nixpkgs)\n' "$nix_version"
+    return 0
+  fi
+
+  printf '%s\n' "$cli_version"
+}
+
+STRFRY_VERSION="$(resolve_strfry_version)"
+NOSTR_BENCH_VERSION="$(nostr-bench --version 2>/dev/null | head -n 1 | tr -d '\r')"
+
+export PARRHESIA_BENCH_CONNECT_COUNT="${PARRHESIA_BENCH_CONNECT_COUNT:-200}"
+export PARRHESIA_BENCH_CONNECT_RATE="${PARRHESIA_BENCH_CONNECT_RATE:-100}"
+export PARRHESIA_BENCH_ECHO_COUNT="${PARRHESIA_BENCH_ECHO_COUNT:-100}"
+export PARRHESIA_BENCH_ECHO_RATE="${PARRHESIA_BENCH_ECHO_RATE:-50}"
+export PARRHESIA_BENCH_ECHO_SIZE="${PARRHESIA_BENCH_ECHO_SIZE:-512}"
+export PARRHESIA_BENCH_EVENT_COUNT="${PARRHESIA_BENCH_EVENT_COUNT:-100}"
+export PARRHESIA_BENCH_EVENT_RATE="${PARRHESIA_BENCH_EVENT_RATE:-50}"
+export PARRHESIA_BENCH_REQ_COUNT="${PARRHESIA_BENCH_REQ_COUNT:-100}"
+export PARRHESIA_BENCH_REQ_RATE="${PARRHESIA_BENCH_REQ_RATE:-50}"
+export PARRHESIA_BENCH_REQ_LIMIT="${PARRHESIA_BENCH_REQ_LIMIT:-10}"
+export PARRHESIA_BENCH_KEEPALIVE_SECONDS="${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}"
+
+WORK_DIR="$(mktemp -d)"
+STRFRY_PID=""
+
+cleanup() {
+  if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then
+    kill "$STRFRY_PID" 2>/dev/null || true
+    wait "$STRFRY_PID" 2>/dev/null || true
+  fi
+
+  if [[ "${KEEP_BENCH_LOGS:-0}" == "1" ]]; then
+    echo "bench logs kept at: $WORK_DIR"
+  else
+    rm -rf "$WORK_DIR"
+  fi
+}
+
+trap cleanup EXIT INT TERM
+
+start_strfry() {
+  local run_index="$1"
+  local port="$2"
+  local strfry_dir="$WORK_DIR/strfry_${run_index}"
+
+  mkdir -p "$strfry_dir/db"
+  cat >"$strfry_dir/strfry.conf" <<EOF
+# generated by scripts/run_bench_compare.sh
+db = "$strfry_dir/db"
+relay {
+  bind = "127.0.0.1"
+  port = ${port}
+  nofiles = 131072
+}
+EOF
+
+  strfry --config "$strfry_dir/strfry.conf" relay >"$strfry_dir/strfry.log" 2>&1 &
+  STRFRY_PID=$!
+
+  for _ in {1..100}; do
+    if ss -ltn | grep -q ":${port} "; then
+      return 0
+    fi
+    sleep 0.1
+  done
+
+  echo "strfry did not become ready on port ${port}" >&2
+  tail -n 120 "$strfry_dir/strfry.log" >&2 || true
+  return 1
+}
+
+stop_strfry() {
+  if [[ -n "$STRFRY_PID" ]] && kill -0 "$STRFRY_PID" 2>/dev/null; then
+    kill "$STRFRY_PID" 2>/dev/null || true
+    wait "$STRFRY_PID" 2>/dev/null || true
+  fi
+  STRFRY_PID=""
+}
+
+echo "Running ${RUNS} comparison run(s)..."
+echo "Versions:"
+echo "  parrhesia ${PARRHESIA_VERSION}"
+echo "  ${STRFRY_VERSION}"
+echo "  ${NOSTR_BENCH_VERSION}"
+echo
+
+for run in $(seq 1 "$RUNS"); do
+  echo "[run ${run}/${RUNS}] Parrhesia"
+  parrhesia_log="$WORK_DIR/parrhesia_${run}.log"
+  if ! ./scripts/run_nostr_bench.sh all >"$parrhesia_log" 2>&1; then
+    echo "Parrhesia benchmark failed. Log: $parrhesia_log" >&2
+    tail -n 120 "$parrhesia_log" >&2 || true
+    exit 1
+  fi
+
+  echo "[run ${run}/${RUNS}] strfry"
+  strfry_log="$WORK_DIR/strfry_${run}.log"
+  strfry_port=$((49000 + run))
+
+  start_strfry "$run" "$strfry_port"
+  if ! STRFRY_BENCH_RELAY_URL="ws://127.0.0.1:${strfry_port}" ./scripts/run_nostr_bench_strfry.sh all >"$strfry_log" 2>&1; then
+    echo "strfry benchmark failed. Log: $strfry_log" >&2
+    tail -n 120 "$strfry_log" >&2 || true
+    stop_strfry
+    exit 1
+  fi
+  stop_strfry
+
+  echo
+
+done
+
+node - "$WORK_DIR" "$RUNS" <<'NODE'
+const fs = require("node:fs");
+const path = require("node:path");
+
+const workDir = process.argv[2];
+const runs = Number(process.argv[3]);
+
+function parseLog(filePath) {
+  const content = fs.readFileSync(filePath, "utf8");
+  let section = null;
+  const last = {};
+
+  for (const lineRaw of content.split(/\r?\n/)) {
+    const line = lineRaw.trim();
+    const match = line.match(/^==>\s+nostr-bench\s+(connect|echo|event|req)\s+/);
+    if (match) {
+      section = match[1];
+      continue;
+    }
+
+    if (!line.startsWith("{")) continue;
+
+    try {
+      const parsed = JSON.parse(line);
+      if (section) {
+        last[section] = parsed;
+      }
+    } catch {
+      // ignore non-json lines
+    }
+  }
+
+  return last;
+}
+
+function runMetrics(parsed) {
+  const connect = parsed.connect?.connect_stats?.success_time ?? {};
+  const echo = parsed.echo ?? {};
+  const event = parsed.event ?? {};
+  const req = parsed.req ?? {};
+
+  return {
+    connectAvgMs: Number(connect.avg ?? NaN),
+    connectMaxMs: Number(connect.max ?? NaN),
+    echoTps: Number(echo.tps ?? NaN),
+    echoSizeMiBS: Number(echo.size ?? NaN),
+    eventTps: Number(event.tps ?? NaN),
+    eventSizeMiBS: Number(event.size ?? NaN),
+    reqTps: Number(req.tps ?? NaN),
+    reqSizeMiBS: Number(req.size ?? NaN),
+  };
+}
+
+function mean(values) {
+  const valid = values.filter((v) => Number.isFinite(v));
+  if (valid.length === 0) return NaN;
+  return valid.reduce((a, b) => a + b, 0) / valid.length;
+}
+
+function toFixed(value, digits = 2) {
+  return Number.isFinite(value) ? value.toFixed(digits) : "n/a";
+}
+
+function loadRuns(prefix) {
+  const all = [];
+  for (let i = 1; i <= runs; i += 1) {
+    const file = path.join(workDir, `${prefix}_${i}.log`);
+    all.push(runMetrics(parseLog(file)));
+  }
+  return all;
+}
+
+const parrhesiaRuns = loadRuns("parrhesia");
+const strfryRuns = loadRuns("strfry");
+
+const summary = {
+  parrhesia: {
+    connectAvgMs: mean(parrhesiaRuns.map((m) => m.connectAvgMs)),
+    connectMaxMs: mean(parrhesiaRuns.map((m) => m.connectMaxMs)),
+    echoTps: mean(parrhesiaRuns.map((m) => m.echoTps)),
+    echoSizeMiBS: mean(parrhesiaRuns.map((m) => m.echoSizeMiBS)),
+    eventTps: mean(parrhesiaRuns.map((m) => m.eventTps)),
+    eventSizeMiBS: mean(parrhesiaRuns.map((m) => m.eventSizeMiBS)),
+    reqTps: mean(parrhesiaRuns.map((m) => m.reqTps)),
+    reqSizeMiBS: mean(parrhesiaRuns.map((m) => m.reqSizeMiBS)),
+  },
+  strfry: {
+    connectAvgMs: mean(strfryRuns.map((m) => m.connectAvgMs)),
+    connectMaxMs: mean(strfryRuns.map((m) => m.connectMaxMs)),
+    echoTps: mean(strfryRuns.map((m) => m.echoTps)),
+    echoSizeMiBS: mean(strfryRuns.map((m) => m.echoSizeMiBS)),
+    eventTps: mean(strfryRuns.map((m) => m.eventTps)),
+    eventSizeMiBS: mean(strfryRuns.map((m) => m.eventSizeMiBS)),
+    reqTps: mean(strfryRuns.map((m) => m.reqTps)),
+    reqSizeMiBS: mean(strfryRuns.map((m) => m.reqSizeMiBS)),
+  },
+};
+
+function ratioStrfryVsParrhesia(metric) {
+  const p = summary.parrhesia[metric];
+  const s = summary.strfry[metric];
+  if (!Number.isFinite(p) || !Number.isFinite(s) || p === 0) return "n/a";
+  return `${(s / p).toFixed(2)}x`;
+}
+
+const rows = [
+  ["connect avg latency (ms) ↓", toFixed(summary.parrhesia.connectAvgMs), toFixed(summary.strfry.connectAvgMs), ratioStrfryVsParrhesia("connectAvgMs")],
+  ["connect max latency (ms) ↓", toFixed(summary.parrhesia.connectMaxMs), toFixed(summary.strfry.connectMaxMs), ratioStrfryVsParrhesia("connectMaxMs")],
+  ["echo throughput (TPS) ↑", toFixed(summary.parrhesia.echoTps), toFixed(summary.strfry.echoTps), ratioStrfryVsParrhesia("echoTps")],
+  ["echo throughput (MiB/s) ↑", toFixed(summary.parrhesia.echoSizeMiBS), toFixed(summary.strfry.echoSizeMiBS), ratioStrfryVsParrhesia("echoSizeMiBS")],
+  ["event throughput (TPS) ↑", toFixed(summary.parrhesia.eventTps), toFixed(summary.strfry.eventTps), ratioStrfryVsParrhesia("eventTps")],
+  ["event throughput (MiB/s) ↑", toFixed(summary.parrhesia.eventSizeMiBS), toFixed(summary.strfry.eventSizeMiBS), ratioStrfryVsParrhesia("eventSizeMiBS")],
+  ["req throughput (TPS) ↑", toFixed(summary.parrhesia.reqTps), toFixed(summary.strfry.reqTps), ratioStrfryVsParrhesia("reqTps")],
+  ["req throughput (MiB/s) ↑", toFixed(summary.parrhesia.reqSizeMiBS), toFixed(summary.strfry.reqSizeMiBS), ratioStrfryVsParrhesia("reqSizeMiBS")],
+];
+
+const headers = ["metric", "parrhesia", "strfry", "strfry/parrhesia"];
+const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
+
+function fmtRow(cols) {
+  return cols
+    .map((c, i) => c.padEnd(widths[i]))
+    .join("  ");
+}
+
+console.log("=== Bench comparison (averages) ===");
+console.log(fmtRow(headers));
+console.log(fmtRow(widths.map((w) => "-".repeat(w))));
+for (const row of rows) {
+  console.log(fmtRow(row));
+}
+
+console.log("\nLegend: ↑ higher is better, ↓ lower is better.");
+console.log("Ratio column is strfry/parrhesia (for ↓ metrics, <1.00x means strfry is faster).\n");
+
+console.log("Run details:");
+for (let i = 0; i < runs; i += 1) {
+  const p = parrhesiaRuns[i];
+  const s = strfryRuns[i];
+  console.log(
+    `  run ${i + 1}: ` +
+      `parrhesia(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)}) | ` +
+      `strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)})`
+  );
+}
+NODE
--- a/scripts/run_nostr_bench.sh
+++ b/scripts/run_nostr_bench.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+usage() {
+  cat <<'EOF'
+usage:
+  ./scripts/run_nostr_bench.sh [all]
+  ./scripts/run_nostr_bench.sh <connect|echo|event|req> [nostr-bench options...]
+
+Runs nostr-bench against a temporary Parrhesia test server started via
+./scripts/run_e2e_suite.sh.
+
+Examples:
+  ./scripts/run_nostr_bench.sh
+  ./scripts/run_nostr_bench.sh connect -c 500 -r 100
+  ./scripts/run_nostr_bench.sh event --json -c 200 -r 100
+
+Default "all" run can be tuned via env vars:
+  PARRHESIA_BENCH_CONNECT_COUNT (default: 200)
+  PARRHESIA_BENCH_CONNECT_RATE  (default: 100)
+  PARRHESIA_BENCH_ECHO_COUNT    (default: 100)
+  PARRHESIA_BENCH_ECHO_RATE     (default: 50)
+  PARRHESIA_BENCH_ECHO_SIZE     (default: 512)
+  PARRHESIA_BENCH_EVENT_COUNT   (default: 100)
+  PARRHESIA_BENCH_EVENT_RATE    (default: 50)
+  PARRHESIA_BENCH_REQ_COUNT     (default: 100)
+  PARRHESIA_BENCH_REQ_RATE      (default: 50)
+  PARRHESIA_BENCH_REQ_LIMIT     (default: 10)
+  PARRHESIA_BENCH_KEEPALIVE_SECONDS (default: 5)
+EOF
+}
+
+MODE="${1:-all}"
+if [[ $# -gt 0 ]]; then
+  shift
+fi
+
+if [[ "$MODE" == "-h" || "$MODE" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if ! command -v nostr-bench >/dev/null 2>&1; then
+  echo "nostr-bench not found in PATH. Enter devenv shell first." >&2
+  exit 1
+fi
+
+if [[ "$MODE" == "all" && $# -gt 0 ]]; then
+  echo "extra arguments are only supported for explicit subcommands" >&2
+  usage
+  exit 1
+fi
+
+exec ./scripts/run_e2e_suite.sh \
+  bench \
+  bash -lc '
+set -euo pipefail
+
+relay_url="ws://127.0.0.1:${PARRHESIA_E2E_RELAY_PORT}/relay"
+mode="$1"
+shift || true
+
+run_default() {
+  echo "==> nostr-bench connect ${relay_url}"
+  nostr-bench connect \
+    --json \
+    -c "${PARRHESIA_BENCH_CONNECT_COUNT:-200}" \
+    -r "${PARRHESIA_BENCH_CONNECT_RATE:-100}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    "${relay_url}"
+
+  echo
+  echo "==> nostr-bench echo ${relay_url}"
+  nostr-bench echo \
+    --json \
+    -c "${PARRHESIA_BENCH_ECHO_COUNT:-100}" \
+    -r "${PARRHESIA_BENCH_ECHO_RATE:-50}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    --size "${PARRHESIA_BENCH_ECHO_SIZE:-512}" \
+    "${relay_url}"
+
+  echo
+  echo "==> nostr-bench event ${relay_url}"
+  nostr-bench event \
+    --json \
+    -c "${PARRHESIA_BENCH_EVENT_COUNT:-100}" \
+    -r "${PARRHESIA_BENCH_EVENT_RATE:-50}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    "${relay_url}"
+
+  echo
+  echo "==> nostr-bench req ${relay_url}"
+  nostr-bench req \
+    --json \
+    -c "${PARRHESIA_BENCH_REQ_COUNT:-100}" \
+    -r "${PARRHESIA_BENCH_REQ_RATE:-50}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    --limit "${PARRHESIA_BENCH_REQ_LIMIT:-10}" \
+    "${relay_url}"
+}
+
+case "$mode" in
+  all)
+    run_default
+    ;;
+  connect | echo | event | req)
+    exec nostr-bench "$mode" "$@" "${relay_url}"
+    ;;
+  *)
+    echo "invalid mode: ${mode}" >&2
+    exit 1
+    ;;
+esac
+' run_nostr_bench "$MODE" "$@"
--- a/scripts/run_nostr_bench_strfry.sh
+++ b/scripts/run_nostr_bench_strfry.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+usage() {
+  cat <<'EOF'
+usage:
+  ./scripts/run_nostr_bench_strfry.sh [all]
+  ./scripts/run_nostr_bench_strfry.sh <connect|echo|event|req> [nostr-bench options...]
+
+Runs nostr-bench against a running strfry relay.
+
+Examples:
+  ./scripts/run_nostr_bench_strfry.sh
+  STRFRY_BENCH_RELAY_URL=ws://127.0.0.1:7777 ./scripts/run_nostr_bench_strfry.sh
+  ./scripts/run_nostr_bench_strfry.sh connect -c 500 -r 100
+
+Relay target env vars:
+  STRFRY_BENCH_RELAY_URL (default: ws://127.0.0.1:${STRFRY_RELAY_PORT:-7777})
+  STRFRY_RELAY_PORT      (default: 7777)
+
+Default "all" run can be tuned via env vars:
+  PARRHESIA_BENCH_CONNECT_COUNT (default: 200)
+  PARRHESIA_BENCH_CONNECT_RATE  (default: 100)
+  PARRHESIA_BENCH_ECHO_COUNT    (default: 100)
+  PARRHESIA_BENCH_ECHO_RATE     (default: 50)
+  PARRHESIA_BENCH_ECHO_SIZE     (default: 512)
+  PARRHESIA_BENCH_EVENT_COUNT   (default: 100)
+  PARRHESIA_BENCH_EVENT_RATE    (default: 50)
+  PARRHESIA_BENCH_REQ_COUNT     (default: 100)
+  PARRHESIA_BENCH_REQ_RATE      (default: 50)
+  PARRHESIA_BENCH_REQ_LIMIT     (default: 10)
+  PARRHESIA_BENCH_KEEPALIVE_SECONDS (default: 5)
+EOF
+}
+
+MODE="${1:-all}"
+if [[ $# -gt 0 ]]; then
+  shift
+fi
+
+if [[ "$MODE" == "-h" || "$MODE" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if ! command -v nostr-bench >/dev/null 2>&1; then
+  echo "nostr-bench not found in PATH. Enter devenv shell first." >&2
+  exit 1
+fi
+
+RELAY_URL="${STRFRY_BENCH_RELAY_URL:-ws://127.0.0.1:${STRFRY_RELAY_PORT:-7777}}"
+
+if [[ "$MODE" == "all" && $# -gt 0 ]]; then
+  echo "extra arguments are only supported for explicit subcommands" >&2
+  usage
+  exit 1
+fi
+
+run_default() {
+  echo "==> nostr-bench connect ${RELAY_URL}"
+  nostr-bench connect \
+    --json \
+    -c "${PARRHESIA_BENCH_CONNECT_COUNT:-200}" \
+    -r "${PARRHESIA_BENCH_CONNECT_RATE:-100}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    "${RELAY_URL}"
+
+  echo
+  echo "==> nostr-bench echo ${RELAY_URL}"
+  nostr-bench echo \
+    --json \
+    -c "${PARRHESIA_BENCH_ECHO_COUNT:-100}" \
+    -r "${PARRHESIA_BENCH_ECHO_RATE:-50}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    --size "${PARRHESIA_BENCH_ECHO_SIZE:-512}" \
+    "${RELAY_URL}"
+
+  echo
+  echo "==> nostr-bench event ${RELAY_URL}"
+  nostr-bench event \
+    --json \
+    -c "${PARRHESIA_BENCH_EVENT_COUNT:-100}" \
+    -r "${PARRHESIA_BENCH_EVENT_RATE:-50}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    "${RELAY_URL}"
+
+  echo
+  echo "==> nostr-bench req ${RELAY_URL}"
+  nostr-bench req \
+    --json \
+    -c "${PARRHESIA_BENCH_REQ_COUNT:-100}" \
+    -r "${PARRHESIA_BENCH_REQ_RATE:-50}" \
+    -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
+    --limit "${PARRHESIA_BENCH_REQ_LIMIT:-10}" \
+    "${RELAY_URL}"
+}
+
+case "$MODE" in
+  all)
+    run_default
+    ;;
+  connect | echo | event | req)
+    exec nostr-bench "$MODE" "$@" "${RELAY_URL}"
+    ;;
+  *)
+    echo "invalid mode: ${MODE}" >&2
+    exit 1
+    ;;
+esac