From fc3d1215993eea02f60773a1b9b911c384304357 Mon Sep 17 00:00:00 2001 From: Steffen Beyer Date: Wed, 18 Mar 2026 21:23:23 +0100 Subject: [PATCH] Benchmark capture and plot --- BENCHMARK.md | 33 -- README.md | 18 +- bench/chart.gnuplot | 31 ++ bench/chart.svg | 752 +++++++++++++++++++++++++++++++++++ bench/history.jsonl | 1 + devenv.nix | 2 + mix.exs | 3 +- scripts/run_bench_compare.sh | 31 ++ scripts/run_bench_update.sh | 329 +++++++++++++++ 9 files changed, 1157 insertions(+), 43 deletions(-) delete mode 100644 BENCHMARK.md create mode 100644 bench/chart.gnuplot create mode 100644 bench/chart.svg create mode 100644 bench/history.jsonl create mode 100755 scripts/run_bench_update.sh diff --git a/BENCHMARK.md b/BENCHMARK.md deleted file mode 100644 index 865b0ef..0000000 --- a/BENCHMARK.md +++ /dev/null @@ -1,33 +0,0 @@ -Running 2 comparison run(s)... -Versions: - parrhesia 0.4.0 - strfry 1.0.4 (nixpkgs) - nostr-rs-relay 0.9.0 - nostr-bench 0.4.0 - -[run 1/2] Parrhesia -[run 1/2] strfry -[run 1/2] nostr-rs-relay - -[run 2/2] Parrhesia -[run 2/2] strfry -[run 2/2] nostr-rs-relay - -=== Bench comparison (averages) === -metric parrhesia strfry nostr-rs-relay strfry/parrhesia nostr-rs/parrhesia --------------------------- --------- -------- -------------- ---------------- ------------------ -connect avg latency (ms) ↓ 10.50 4.00 3.00 0.38x 0.29x -connect max latency (ms) ↓ 19.50 7.50 4.00 0.38x 0.21x -echo throughput (TPS) ↑ 78520.00 60353.00 164420.50 0.77x 2.09x -echo throughput (MiB/s) ↑ 43.00 33.75 90.05 0.78x 2.09x -event throughput (TPS) ↑ 1919.50 3520.50 781.00 1.83x 0.41x -event throughput (MiB/s) ↑ 1.25 2.25 0.50 1.80x 0.40x -req throughput (TPS) ↑ 4608.50 1809.50 875.50 0.39x 0.19x -req throughput (MiB/s) ↑ 26.20 11.75 2.40 0.45x 0.09x - -Legend: ↑ higher is better, ↓ lower is better. -Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster). - -Run details: - run 1: parrhesia(echo_tps=78892, event_tps=1955, req_tps=4671, connect_avg_ms=10) | strfry(echo_tps=59132, event_tps=3462, req_tps=1806, connect_avg_ms=4) | nostr-rs-relay(echo_tps=159714, event_tps=785, req_tps=873, connect_avg_ms=3) - run 2: parrhesia(echo_tps=78148, event_tps=1884, req_tps=4546, connect_avg_ms=11) | strfry(echo_tps=61574, event_tps=3579, req_tps=1813, connect_avg_ms=4) | nostr-rs-relay(echo_tps=169127, event_tps=777, req_tps=878, connect_avg_ms=3) diff --git a/README.md b/README.md index b81ea8e..99d7a28 100644 --- a/README.md +++ b/README.md @@ -550,16 +550,16 @@ mix bench Current comparison results from [BENCHMARK.md](./BENCHMARK.md): -| metric | parrhesia | strfry | nostr-rs-relay | strfry/parrhesia | nostr-rs/parrhesia | +| metric | parrhesia-pg | parrhesia-mem | nostr-rs-relay | mem/pg | nostr-rs/pg | | --- | ---: | ---: | ---: | ---: | ---: | -| connect avg latency (ms) ↓ | 13.50 | 3.00 | 2.00 | **0.22x** | **0.15x** | -| connect max latency (ms) ↓ | 22.50 | 5.50 | 3.00 | **0.24x** | **0.13x** | -| echo throughput (TPS) ↑ | 80385.00 | 61673.00 | 164516.00 | 0.77x | **2.05x** | -| echo throughput (MiB/s) ↑ | 44.00 | 34.45 | 90.10 | 0.78x | **2.05x** | -| event throughput (TPS) ↑ | 2000.00 | 3404.50 | 788.00 | **1.70x** | 0.39x | -| event throughput (MiB/s) ↑ | 1.30 | 2.20 | 0.50 | **1.69x** | 0.38x | -| req throughput (TPS) ↑ | 3664.00 | 1808.50 | 877.50 | 0.49x | 0.24x | -| req throughput (MiB/s) ↑ | 20.75 | 11.75 | 2.45 | 0.57x | 0.12x | +| connect avg latency (ms) ↓ | 9.33 | 7.67 | 7.00 | **0.82x** | **0.75x** | +| connect max latency (ms) ↓ | 12.33 | 9.67 | 10.33 | **0.78x** | **0.84x** | +| echo throughput (TPS) ↑ | 64030.33 | 93656.33 | 140767.00 | **1.46x** | **2.20x** | +| echo throughput (MiB/s) ↑ | 35.07 | 51.27 | 77.07 | **1.46x** | **2.20x** | +| event throughput (TPS) ↑ | 5015.33 | 1505.33 | 2293.67 | 0.30x | 0.46x | +| event throughput (MiB/s) ↑ | 3.40 | 1.00 | 1.50 | 0.29x | 0.44x | +| req throughput (TPS) ↑ | 6416.33 | 14566.67 | 3035.67 | **2.27x** | 0.47x | +| req throughput (MiB/s) ↑ | 42.43 | 94.23 | 19.23 | **2.22x** | 0.45x | Higher is better for `↑` metrics. Lower is better for `↓` metrics. diff --git a/bench/chart.gnuplot b/bench/chart.gnuplot new file mode 100644 index 0000000..6f8bc38 --- /dev/null +++ b/bench/chart.gnuplot @@ -0,0 +1,31 @@ +# bench/chart.gnuplot — multi-panel SVG showing relay performance over git tags. +# +# Invoked by scripts/run_bench_update.sh with: +# gnuplot -e "data_dir='...'" -e "output_file='...'" bench/chart.gnuplot +# +# The data_dir contains per-metric TSV files and a plot_commands.gnuplot +# fragment generated by the data-prep step that defines the actual plot +# directives (handling variable server columns). + +set terminal svg enhanced size 1200,900 font "sans,11" +set output output_file + +set style data linespoints +set key outside right top +set grid ytics +set xtics rotate by -30 +set datafile separator "\t" + +# parrhesia-pg: blue solid, parrhesia-memory: green solid +# strfry: orange dashed, nostr-rs-relay: red dashed +set linetype 1 lc rgb "#2563eb" lw 2 pt 7 ps 1.0 +set linetype 2 lc rgb "#16a34a" lw 2 pt 9 ps 1.0 +set linetype 3 lc rgb "#ea580c" lw 1.5 pt 5 ps 0.8 dt 2 +set linetype 4 lc rgb "#dc2626" lw 1.5 pt 4 ps 0.8 dt 2 + +set multiplot layout 2,2 title "Parrhesia Relay Benchmark History" font ",14" + +# Load dynamically generated plot commands (handles variable column counts) +load data_dir."/plot_commands.gnuplot" + +unset multiplot diff --git a/bench/chart.svg b/bench/chart.svg new file mode 100644 index 0000000..8576853 --- /dev/null +++ b/bench/chart.svg @@ -0,0 +1,752 @@ + + + +Gnuplot +Produced by GNUPLOT 6.0 patchlevel 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Parrhesia Relay Benchmark History + + + + + + + + + + + + + + + 1500 + + + + + + + + + + + + + 2000 + + + + + + + + + + + + + 2500 + + + + + + + + + + + + + 3000 + + + + + + + + + + + + + 3500 + + + + + + + + + + + + + 4000 + + + + + + + + + + + + + 4500 + + + + + + + + + + + + + 5000 + + + + + + + + + + + + + 5500 + + + + + v0.5.0 + + + + + + + + + parrhesia-pg + + + + + parrhesia-pg + + + + + + + + parrhesia-memory + + + parrhesia-memory + + + + + + + + nostr-rs-relay (avg) + + + + + nostr-rs-relay (avg) + + + + + + + + + + + + + + + + TPS + + + + + + + Event Throughput (TPS) — higher is better + + + + + + + + + + + + + + + 2000 + + + + + + + + + + + + + 4000 + + + + + + + + + + + + + 6000 + + + + + + + + + + + + + 8000 + + + + + + + + + + + + + 10000 + + + + + + + + + + + + + 12000 + + + + + + + + + + + + + 14000 + + + + + + + + + + + + + 16000 + + + + + v0.5.0 + + + + + + + + + parrhesia-pg + + + + + parrhesia-pg + + + + + + + + parrhesia-memory + + + parrhesia-memory + + + + + + + + nostr-rs-relay (avg) + + + + + nostr-rs-relay (avg) + + + + + + + + + + + + + + + + TPS + + + + + + + Req Throughput (TPS) — higher is better + + + + + + + + + + + + + + + 60000 + + + + + + + + + + + + + 70000 + + + + + + + + + + + + + 80000 + + + + + + + + + + + + + 90000 + + + + + + + + + + + + + 100000 + + + + + + + + + + + + + 110000 + + + + + + + + + + + + + 120000 + + + + + + + + + + + + + 130000 + + + + + + + + + + + + + 140000 + + + + + + + + + + + + + 150000 + + + + + v0.5.0 + + + + + + + + + parrhesia-pg + + + + + parrhesia-pg + + + + + + + + parrhesia-memory + + + parrhesia-memory + + + + + + + + nostr-rs-relay (avg) + + + + + nostr-rs-relay (avg) + + + + + + + + + + + + + + + + TPS + + + + + + + Echo Throughput (TPS) — higher is better + + + + + + + + + + + + + + + 7 + + + + + + + + + + + + + 7.5 + + + + + + + + + + + + + 8 + + + + + + + + + + + + + 8.5 + + + + + + + + + + + + + 9 + + + + + + + + + + + + + 9.5 + + + + + v0.5.0 + + + + + + + + + parrhesia-pg + + + + + parrhesia-pg + + + + + + + + parrhesia-memory + + + parrhesia-memory + + + + + + + + nostr-rs-relay (avg) + + + + + nostr-rs-relay (avg) + + + + + + + + + + + + + + + + ms + + + + + + + Connect Avg Latency (ms) — lower is better + + + + + + + diff --git a/bench/history.jsonl b/bench/history.jsonl new file mode 100644 index 0000000..98dac89 --- /dev/null +++ b/bench/history.jsonl @@ -0,0 +1 @@ +{"timestamp":"2026-03-18T20:13:21Z","machine_id":"squirrel","git_tag":"v0.5.0","git_commit":"970cee2","runs":3,"servers":{"parrhesia-pg":{"connect_avg_ms":9.333333333333334,"connect_max_ms":12.333333333333334,"echo_tps":64030.333333333336,"echo_mibs":35.06666666666666,"event_tps":5015.333333333333,"event_mibs":3.4,"req_tps":6416.333333333333,"req_mibs":42.43333333333334},"parrhesia-memory":{"connect_avg_ms":7.666666666666667,"connect_max_ms":9.666666666666666,"echo_tps":93656.33333333333,"echo_mibs":51.26666666666667,"event_tps":1505.3333333333333,"event_mibs":1,"req_tps":14566.666666666666,"req_mibs":94.23333333333335},"nostr-rs-relay":{"connect_avg_ms":7,"connect_max_ms":10.333333333333334,"echo_tps":140767,"echo_mibs":77.06666666666666,"event_tps":2293.6666666666665,"event_mibs":1.5,"req_tps":3035.6666666666665,"req_mibs":19.23333333333333}}} diff --git a/devenv.nix b/devenv.nix index c76e5e3..260fac7 100644 --- a/devenv.nix +++ b/devenv.nix @@ -101,6 +101,8 @@ in { nostr-bench # Nostr reference servers nostr-rs-relay + # Benchmark graph + gnuplot ] ++ lib.optionals pkgs.stdenv.hostPlatform.isx86_64 [ strfry diff --git a/mix.exs b/mix.exs index d053292..59c51a9 100644 --- a/mix.exs +++ b/mix.exs @@ -26,7 +26,7 @@ defmodule Parrhesia.MixProject do defp elixirc_paths(_env), do: ["lib"] def cli do - [preferred_envs: [precommit: :test, bench: :test]] + [preferred_envs: [precommit: :test, bench: :test, "bench.update": :test]] end # Run "mix help deps" to learn about dependencies. @@ -71,6 +71,7 @@ defmodule Parrhesia.MixProject do "test.node_sync_e2e": ["cmd ./scripts/run_node_sync_e2e.sh"], "test.node_sync_docker_e2e": ["cmd ./scripts/run_node_sync_docker_e2e.sh"], bench: ["cmd ./scripts/run_bench_compare.sh"], + "bench.update": ["cmd ./scripts/run_bench_update.sh"], # cov: ["cmd mix coveralls.lcov"], lint: ["format --check-formatted", "credo"], precommit: [ diff --git a/scripts/run_bench_compare.sh b/scripts/run_bench_compare.sh index 177f34a..094bef4 100755 --- a/scripts/run_bench_compare.sh +++ b/scripts/run_bench_compare.sh @@ -477,4 +477,35 @@ for (let i = 0; i < runs; i += 1) { } console.log(line); } + +// Structured JSON output for automation (bench:update pipeline) +if (process.env.BENCH_JSON_OUT) { + const jsonSummary = {}; + const serverKeys = [ + ["parrhesia-pg", "parrhesia"], + ["parrhesia-memory", "parrhesiaMemory"], + ]; + if (hasStrfry) serverKeys.push(["strfry", "strfry"]); + if (hasNostrRs) serverKeys.push(["nostr-rs-relay", "nostrRsRelay"]); + + for (const [outputKey, summaryKey] of serverKeys) { + const s = summary[summaryKey]; + jsonSummary[outputKey] = { + connect_avg_ms: s.connectAvgMs, + connect_max_ms: s.connectMaxMs, + echo_tps: s.echoTps, + echo_mibs: s.echoSizeMiBS, + event_tps: s.eventTps, + event_mibs: s.eventSizeMiBS, + req_tps: s.reqTps, + req_mibs: s.reqSizeMiBS, + }; + } + + fs.writeFileSync( + process.env.BENCH_JSON_OUT, + JSON.stringify(jsonSummary, null, 2) + "\n", + "utf8" + ); +} NODE diff --git a/scripts/run_bench_update.sh b/scripts/run_bench_update.sh new file mode 100755 index 0000000..7fa06d3 --- /dev/null +++ b/scripts/run_bench_update.sh @@ -0,0 +1,329 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +usage() { + cat <<'EOF' +usage: + ./scripts/run_bench_update.sh + +Runs the benchmark suite (3 runs by default), then: + 1) Appends structured results to bench/history.jsonl + 2) Generates bench/chart.svg via gnuplot + 3) Updates the comparison table in README.md + +Environment: + PARRHESIA_BENCH_RUNS Number of runs (default: 3) + PARRHESIA_BENCH_MACHINE_ID Machine identifier (default: hostname -s) + +All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded. +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +# --- Configuration ----------------------------------------------------------- + +BENCH_DIR="$ROOT_DIR/bench" +HISTORY_FILE="$BENCH_DIR/history.jsonl" +CHART_FILE="$BENCH_DIR/chart.svg" +GNUPLOT_TEMPLATE="$BENCH_DIR/chart.gnuplot" + +MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}" +GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')" +GIT_COMMIT="$(git rev-parse --short=7 HEAD)" +TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +RUNS="${PARRHESIA_BENCH_RUNS:-3}" + +mkdir -p "$BENCH_DIR" + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +JSON_OUT="$WORK_DIR/bench_summary.json" +RAW_OUTPUT="$WORK_DIR/bench_output.txt" + +# --- Phase 1: Run benchmarks ------------------------------------------------- + +echo "Running ${RUNS}-run benchmark suite..." + +PARRHESIA_BENCH_RUNS="$RUNS" \ +BENCH_JSON_OUT="$JSON_OUT" \ + ./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT" + +if [[ ! -f "$JSON_OUT" ]]; then + echo "Benchmark JSON output not found at $JSON_OUT" >&2 + exit 1 +fi + +# --- Phase 2: Append to history ---------------------------------------------- + +echo "Appending to history..." + +node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE' +const fs = require("node:fs"); + +const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv; + +const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8")); + +const entry = { + timestamp, + machine_id: machineId, + git_tag: gitTag, + git_commit: gitCommit, + runs: Number(runsStr), + servers, +}; + +fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8"); +console.log(" entry: " + gitTag + " (" + gitCommit + ") on " + machineId); +NODE + +# --- Phase 3: Generate chart -------------------------------------------------- + +echo "Generating chart..." + +node - "$HISTORY_FILE" "$MACHINE_ID" "$WORK_DIR" <<'NODE' +const fs = require("node:fs"); +const path = require("node:path"); + +const [, , historyFile, machineId, workDir] = process.argv; + +if (!fs.existsSync(historyFile)) { + console.log(" no history file, skipping chart generation"); + process.exit(0); +} + +const lines = fs.readFileSync(historyFile, "utf8") + .split("\n") + .filter(l => l.trim().length > 0) + .map(l => JSON.parse(l)); + +// Filter to current machine +const entries = lines.filter(e => e.machine_id === machineId); + +if (entries.length === 0) { + console.log(" no history entries for machine '" + machineId + "', skipping chart"); + process.exit(0); +} + +// Sort chronologically, deduplicate by tag (latest wins) +entries.sort((a, b) => a.timestamp.localeCompare(b.timestamp)); +const byTag = new Map(); +for (const e of entries) { + byTag.set(e.git_tag, e); +} +const deduped = [...byTag.values()]; + +// Determine which non-parrhesia servers are present +const baselineServerNames = ["strfry", "nostr-rs-relay"]; +const presentBaselines = baselineServerNames.filter(srv => + deduped.some(e => e.servers[srv]) +); + +// Compute averages for baseline servers (constant horizontal lines) +const baselineAvg = {}; +for (const srv of presentBaselines) { + const vals = deduped.filter(e => e.servers[srv]).map(e => e.servers[srv]); + baselineAvg[srv] = {}; + for (const metric of Object.keys(vals[0])) { + const valid = vals.map(v => v[metric]).filter(Number.isFinite); + baselineAvg[srv][metric] = valid.length > 0 + ? valid.reduce((a, b) => a + b, 0) / valid.length + : NaN; + } +} + +// Metrics to chart +const chartMetrics = [ + { key: "event_tps", label: "Event Throughput (TPS) — higher is better", file: "event_tps.tsv", ylabel: "TPS" }, + { key: "req_tps", label: "Req Throughput (TPS) — higher is better", file: "req_tps.tsv", ylabel: "TPS" }, + { key: "echo_tps", label: "Echo Throughput (TPS) — higher is better", file: "echo_tps.tsv", ylabel: "TPS" }, + { key: "connect_avg_ms", label: "Connect Avg Latency (ms) — lower is better", file: "connect_avg_ms.tsv", ylabel: "ms" }, +]; + +// Write per-metric TSV files +for (const cm of chartMetrics) { + const header = ["tag", "parrhesia-pg", "parrhesia-memory"]; + for (const srv of presentBaselines) header.push(srv); + + const rows = [header.join("\t")]; + for (const e of deduped) { + const row = [ + e.git_tag, + e.servers["parrhesia-pg"]?.[cm.key] ?? "NaN", + e.servers["parrhesia-memory"]?.[cm.key] ?? "NaN", + ]; + for (const srv of presentBaselines) { + row.push(baselineAvg[srv]?.[cm.key] ?? "NaN"); + } + rows.push(row.join("\t")); + } + + fs.writeFileSync(path.join(workDir, cm.file), rows.join("\n") + "\n", "utf8"); +} + +// Generate gnuplot plot commands (handles variable column counts) +const serverLabels = ["parrhesia-pg", "parrhesia-memory"]; +for (const srv of presentBaselines) serverLabels.push(srv + " (avg)"); + +const plotLines = []; +for (const cm of chartMetrics) { + const dataFile = `data_dir."/${cm.file}"`; + plotLines.push(`set title "${cm.label}"`); + plotLines.push(`set ylabel "${cm.ylabel}"`); + + const plotParts = []; + // Column 2 = parrhesia-pg, 3 = parrhesia-memory, 4+ = baselines + plotParts.push(`${dataFile} using 0:2:xtic(1) lt 1 title "${serverLabels[0]}"`); + plotParts.push(`'' using 0:3 lt 2 title "${serverLabels[1]}"`); + for (let i = 0; i < presentBaselines.length; i++) { + plotParts.push(`'' using 0:${4 + i} lt ${3 + i} title "${serverLabels[2 + i]}"`); + } + + plotLines.push("plot " + plotParts.join(", \\\n ")); + plotLines.push(""); +} + +fs.writeFileSync( + path.join(workDir, "plot_commands.gnuplot"), + plotLines.join("\n") + "\n", + "utf8" +); + +console.log(" " + deduped.length + " tag(s), " + presentBaselines.length + " baseline server(s)"); +NODE + +if [[ -f "$WORK_DIR/plot_commands.gnuplot" ]]; then + gnuplot \ + -e "data_dir='$WORK_DIR'" \ + -e "output_file='$CHART_FILE'" \ + "$GNUPLOT_TEMPLATE" + echo " chart written to $CHART_FILE" +else + echo " chart generation skipped (no data for this machine)" +fi + +# --- Phase 4: Update README.md ----------------------------------------------- + +echo "Updating README.md..." + +node - "$JSON_OUT" "$ROOT_DIR/README.md" <<'NODE' +const fs = require("node:fs"); + +const [, , jsonOut, readmePath] = process.argv; + +const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8")); +const readme = fs.readFileSync(readmePath, "utf8"); + +const pg = servers["parrhesia-pg"]; +const mem = servers["parrhesia-memory"]; +const strfry = servers["strfry"]; +const nostrRs = servers["nostr-rs-relay"]; + +function toFixed(v, d = 2) { + return Number.isFinite(v) ? v.toFixed(d) : "n/a"; +} + +function ratio(base, other) { + if (!Number.isFinite(base) || !Number.isFinite(other) || base === 0) return "n/a"; + return (other / base).toFixed(2) + "x"; +} + +function boldIf(ratioStr, lowerIsBetter) { + if (ratioStr === "n/a") return ratioStr; + const num = parseFloat(ratioStr); + const better = lowerIsBetter ? num < 1 : num > 1; + return better ? "**" + ratioStr + "**" : ratioStr; +} + +const metricRows = [ + ["connect avg latency (ms) \u2193", "connect_avg_ms", true], + ["connect max latency (ms) \u2193", "connect_max_ms", true], + ["echo throughput (TPS) \u2191", "echo_tps", false], + ["echo throughput (MiB/s) \u2191", "echo_mibs", false], + ["event throughput (TPS) \u2191", "event_tps", false], + ["event throughput (MiB/s) \u2191", "event_mibs", false], + ["req throughput (TPS) \u2191", "req_tps", false], + ["req throughput (MiB/s) \u2191", "req_mibs", false], +]; + +const hasStrfry = !!strfry; +const hasNostrRs = !!nostrRs; + +// Build header +const header = ["metric", "parrhesia-pg", "parrhesia-mem"]; +if (hasStrfry) header.push("strfry"); +if (hasNostrRs) header.push("nostr-rs-relay"); +header.push("mem/pg"); +if (hasStrfry) header.push("strfry/pg"); +if (hasNostrRs) header.push("nostr-rs/pg"); + +const alignRow = ["---"]; +for (let i = 1; i < header.length; i++) alignRow.push("---:"); + +const rows = metricRows.map(([label, key, lowerIsBetter]) => { + const row = [label, toFixed(pg[key]), toFixed(mem[key])]; + if (hasStrfry) row.push(toFixed(strfry[key])); + if (hasNostrRs) row.push(toFixed(nostrRs[key])); + + row.push(boldIf(ratio(pg[key], mem[key]), lowerIsBetter)); + if (hasStrfry) row.push(boldIf(ratio(pg[key], strfry[key]), lowerIsBetter)); + if (hasNostrRs) row.push(boldIf(ratio(pg[key], nostrRs[key]), lowerIsBetter)); + + return row; +}); + +const tableLines = [ + "| " + header.join(" | ") + " |", + "| " + alignRow.join(" | ") + " |", + ...rows.map(r => "| " + r.join(" | ") + " |"), +]; + +// Replace the first markdown table in the ## Benchmark section +const readmeLines = readme.split("\n"); +const benchIdx = readmeLines.findIndex(l => /^## Benchmark/.test(l)); +if (benchIdx === -1) { + console.error("Could not find '## Benchmark' section in README.md"); + process.exit(1); +} + +let tableStart = -1; +let tableEnd = -1; +for (let i = benchIdx + 1; i < readmeLines.length; i++) { + if (readmeLines[i].startsWith("|")) { + if (tableStart === -1) tableStart = i; + tableEnd = i; + } else if (tableStart !== -1) { + break; + } +} + +if (tableStart === -1) { + console.error("Could not find markdown table in ## Benchmark section"); + process.exit(1); +} + +const before = readmeLines.slice(0, tableStart); +const after = readmeLines.slice(tableEnd + 1); +const updated = [...before, ...tableLines, ...after].join("\n"); + +fs.writeFileSync(readmePath, updated, "utf8"); +console.log(" table updated (" + tableLines.length + " rows)"); +NODE + +# --- Done --------------------------------------------------------------------- + +echo +echo "Benchmark update complete. Files changed:" +echo " $HISTORY_FILE" +echo " $CHART_FILE" +echo " $ROOT_DIR/README.md" +echo +echo "Review with: git diff"