From 7c0ad28f6e761e8a855b49666962af05941665e9 Mon Sep 17 00:00:00 2001 From: Steffen Beyer Date: Wed, 18 Mar 2026 22:10:11 +0100 Subject: [PATCH] bench: Capture versions --- README.md | 20 +- bench/chart.svg | 496 ++++++++++++++++++----------------- bench/history.jsonl | 2 +- scripts/run_bench_compare.sh | 13 +- scripts/run_bench_update.sh | 22 +- 5 files changed, 285 insertions(+), 268 deletions(-) diff --git a/README.md b/README.md index 2cc074b..07534b2 100644 --- a/README.md +++ b/README.md @@ -550,16 +550,16 @@ mix bench Current comparison results from [BENCHMARK.md](./BENCHMARK.md): -| metric | parrhesia-pg | parrhesia-mem | nostr-rs-relay | mem/pg | nostr-rs/pg | -| --- | ---: | ---: | ---: | ---: | ---: | -| connect avg latency (ms) ↓ | 9.33 | 7.67 | 7.00 | **0.82x** | **0.75x** | -| connect max latency (ms) ↓ | 12.33 | 9.67 | 10.33 | **0.78x** | **0.84x** | -| echo throughput (TPS) ↑ | 64030.33 | 93656.33 | 140767.00 | **1.46x** | **2.20x** | -| echo throughput (MiB/s) ↑ | 35.07 | 51.27 | 77.07 | **1.46x** | **2.20x** | -| event throughput (TPS) ↑ | 5015.33 | 1505.33 | 2293.67 | 0.30x | 0.46x | -| event throughput (MiB/s) ↑ | 3.40 | 1.00 | 1.50 | 0.29x | 0.44x | -| req throughput (TPS) ↑ | 6416.33 | 14566.67 | 3035.67 | **2.27x** | 0.47x | -| req throughput (MiB/s) ↑ | 42.43 | 94.23 | 19.23 | **2.22x** | 0.45x | +| metric | parrhesia-pg | parrhesia-mem | strfry | nostr-rs-relay | mem/pg | strfry/pg | nostr-rs/pg | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| connect avg latency (ms) ↓ | 26.67 | 14.67 | 3.00 | 2.00 | **0.55x** | **0.11x** | **0.07x** | +| connect max latency (ms) ↓ | 45.33 | 24.33 | 4.67 | 3.33 | **0.54x** | **0.10x** | **0.07x** | +| echo throughput (TPS) ↑ | 68100.33 | 55978.00 | 67718.33 | 166178.00 | 0.82x | 0.99x | **2.44x** | +| echo throughput (MiB/s) ↑ | 37.23 | 30.63 | 37.87 | 91.03 | 0.82x | **1.02x** | **2.44x** | +| event throughput (TPS) ↑ | 1647.33 | 882.00 | 3548.33 | 787.00 | 0.54x | **2.15x** | 0.48x | +| event throughput (MiB/s) ↑ | 1.07 | 0.57 | 2.30 | 0.50 | 0.53x | **2.16x** | 0.47x | +| req throughput (TPS) ↑ | 3576.67 | 6888.00 | 1808.00 | 860.67 | **1.93x** | 0.51x | 0.24x | +| req throughput (MiB/s) ↑ | 18.83 | 36.07 | 11.70 | 2.40 | **1.92x** | 0.62x | 0.13x | Higher is better for `↑` metrics. Lower is better for `↓` metrics. diff --git a/bench/chart.svg b/bench/chart.svg index 8576853..f993cdf 100644 --- a/bench/chart.svg +++ b/bench/chart.svg @@ -51,11 +51,37 @@ - + - + + 500 + + + + + + + + + + + + + 1000 + + + + + + + + + + + + 1500 @@ -64,11 +90,11 @@ - + - + 2000 @@ -77,11 +103,11 @@ - + - + 2500 @@ -90,11 +116,11 @@ - + - + 3000 @@ -103,11 +129,11 @@ - + - + 3500 @@ -116,62 +142,23 @@ - + - + 4000 - - - - - - - - - - 4500 - - - - - - - - - - - - - 5000 - - - - - - - - - - - - - 5500 - - - - - v0.5.0 + + v0.6.0 - + parrhesia-pg @@ -183,7 +170,7 @@ - + @@ -194,38 +181,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + TPS - + Event Throughput (TPS) — higher is better @@ -236,11 +234,37 @@ - + - + + 0 + + + + + + + + + + + + + 1000 + + + + + + + + + + + + 2000 @@ -249,11 +273,24 @@ - + - + + 3000 + + + + + + + + + + + + 4000 @@ -262,11 +299,24 @@ - + - + + 5000 + + + + + + + + + + + + 6000 @@ -275,75 +325,23 @@ - + - - 8000 + + 7000 - - - - - - - - - - 10000 - - - - - - - - - - - - - 12000 - - - - - - - - - - - - - 14000 - - - - - - - - - - - - - 16000 - - - - - v0.5.0 + + v0.6.0 - + parrhesia-pg @@ -355,7 +353,7 @@ - + @@ -366,38 +364,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + TPS - + Req Throughput (TPS) — higher is better @@ -408,11 +417,24 @@ - + - + + 40000 + + + + + + + + + + + + 60000 @@ -421,24 +443,11 @@ - + - - 70000 - - - - - - - - - - - - + 80000 @@ -447,24 +456,11 @@ - + - - 90000 - - - - - - - - - - - - + 100000 @@ -473,24 +469,11 @@ - + - - 110000 - - - - - - - - - - - - + 120000 @@ -499,24 +482,11 @@ - + - - 130000 - - - - - - - - - - - - + 140000 @@ -525,23 +495,36 @@ - + - - 150000 + + 160000 - - v0.5.0 + + + + + + + + + + 180000 + + + + + v0.6.0 - + parrhesia-pg @@ -553,7 +536,7 @@ - + @@ -564,38 +547,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + TPS - + Echo Throughput (TPS) — higher is better @@ -606,12 +600,12 @@ - + - - 7 + + 0 @@ -619,12 +613,12 @@ - + - - 7.5 + + 5 @@ -632,12 +626,12 @@ - + - - 8 + + 10 @@ -645,12 +639,12 @@ - + - - 8.5 + + 15 @@ -658,12 +652,12 @@ - + - - 9 + + 20 @@ -671,23 +665,36 @@ - + - - 9.5 + + 25 - - v0.5.0 + + + + + + + + + + 30 + + + + + v0.6.0 - + parrhesia-pg @@ -699,7 +706,7 @@ - + @@ -710,38 +717,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + ms - + Connect Avg Latency (ms) — lower is better diff --git a/bench/history.jsonl b/bench/history.jsonl index 98dac89..738d9b1 100644 --- a/bench/history.jsonl +++ b/bench/history.jsonl @@ -1 +1 @@ -{"timestamp":"2026-03-18T20:13:21Z","machine_id":"squirrel","git_tag":"v0.5.0","git_commit":"970cee2","runs":3,"servers":{"parrhesia-pg":{"connect_avg_ms":9.333333333333334,"connect_max_ms":12.333333333333334,"echo_tps":64030.333333333336,"echo_mibs":35.06666666666666,"event_tps":5015.333333333333,"event_mibs":3.4,"req_tps":6416.333333333333,"req_mibs":42.43333333333334},"parrhesia-memory":{"connect_avg_ms":7.666666666666667,"connect_max_ms":9.666666666666666,"echo_tps":93656.33333333333,"echo_mibs":51.26666666666667,"event_tps":1505.3333333333333,"event_mibs":1,"req_tps":14566.666666666666,"req_mibs":94.23333333333335},"nostr-rs-relay":{"connect_avg_ms":7,"connect_max_ms":10.333333333333334,"echo_tps":140767,"echo_mibs":77.06666666666666,"event_tps":2293.6666666666665,"event_mibs":1.5,"req_tps":3035.6666666666665,"req_mibs":19.23333333333333}}} +{"timestamp":"2026-03-18T21:35:03Z","machine_id":"agent","git_tag":"v0.6.0","git_commit":"7b337d9","runs":3,"versions":{"parrhesia":"0.6.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":26.666666666666668,"connect_max_ms":45.333333333333336,"echo_tps":68100.33333333333,"echo_mibs":37.233333333333334,"event_tps":1647.3333333333333,"event_mibs":1.0666666666666667,"req_tps":3576.6666666666665,"req_mibs":18.833333333333332},"parrhesia-memory":{"connect_avg_ms":14.666666666666666,"connect_max_ms":24.333333333333332,"echo_tps":55978,"echo_mibs":30.633333333333336,"event_tps":882,"event_mibs":0.5666666666666668,"req_tps":6888,"req_mibs":36.06666666666666},"strfry":{"connect_avg_ms":3,"connect_max_ms":4.666666666666667,"echo_tps":67718.33333333333,"echo_mibs":37.86666666666667,"event_tps":3548.3333333333335,"event_mibs":2.3,"req_tps":1808,"req_mibs":11.699999999999998},"nostr-rs-relay":{"connect_avg_ms":2,"connect_max_ms":3.3333333333333335,"echo_tps":166178,"echo_mibs":91.03333333333335,"event_tps":787,"event_mibs":0.5,"req_tps":860.6666666666666,"req_mibs":2.4}}} diff --git a/scripts/run_bench_compare.sh b/scripts/run_bench_compare.sh index 094bef4..f0e15c9 100755 --- a/scripts/run_bench_compare.sh +++ b/scripts/run_bench_compare.sh @@ -298,7 +298,8 @@ for run in $(seq 1 "$RUNS"); do done -node - "$WORK_DIR" "$RUNS" "$HAS_STRFRY" "$HAS_NOSTR_RS" <<'NODE' +node - "$WORK_DIR" "$RUNS" "$HAS_STRFRY" "$HAS_NOSTR_RS" \ + "$PARRHESIA_VERSION" "$STRFRY_VERSION" "$NOSTR_RS_RELAY_VERSION" "$NOSTR_BENCH_VERSION" <<'NODE' const fs = require("node:fs"); const path = require("node:path"); @@ -306,6 +307,10 @@ const workDir = process.argv[2]; const runs = Number(process.argv[3]); const hasStrfry = process.argv[4] === "1"; const hasNostrRs = process.argv[5] === "1"; +const parrhesiaVersion = process.argv[6] || ""; +const strfryVersion = process.argv[7] || ""; +const nostrRsRelayVersion = process.argv[8] || ""; +const nostrBenchVersion = process.argv[9] || ""; function parseLog(filePath) { const content = fs.readFileSync(filePath, "utf8"); @@ -502,6 +507,12 @@ if (process.env.BENCH_JSON_OUT) { }; } + const versions = { parrhesia: parrhesiaVersion }; + if (hasStrfry && strfryVersion) versions.strfry = strfryVersion; + if (hasNostrRs && nostrRsRelayVersion) versions["nostr-rs-relay"] = nostrRsRelayVersion; + if (nostrBenchVersion) versions["nostr-bench"] = nostrBenchVersion; + jsonSummary.versions = versions; + fs.writeFileSync( process.env.BENCH_JSON_OUT, JSON.stringify(jsonSummary, null, 2) + "\n", diff --git a/scripts/run_bench_update.sh b/scripts/run_bench_update.sh index 7fa06d3..06e011f 100755 --- a/scripts/run_bench_update.sh +++ b/scripts/run_bench_update.sh @@ -70,7 +70,7 @@ const fs = require("node:fs"); const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv; -const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8")); +const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8")); const entry = { timestamp, @@ -78,6 +78,7 @@ const entry = { git_tag: gitTag, git_commit: gitCommit, runs: Number(runsStr), + versions: versions || {}, servers, }; @@ -127,19 +128,6 @@ const presentBaselines = baselineServerNames.filter(srv => deduped.some(e => e.servers[srv]) ); -// Compute averages for baseline servers (constant horizontal lines) -const baselineAvg = {}; -for (const srv of presentBaselines) { - const vals = deduped.filter(e => e.servers[srv]).map(e => e.servers[srv]); - baselineAvg[srv] = {}; - for (const metric of Object.keys(vals[0])) { - const valid = vals.map(v => v[metric]).filter(Number.isFinite); - baselineAvg[srv][metric] = valid.length > 0 - ? valid.reduce((a, b) => a + b, 0) / valid.length - : NaN; - } -} - // Metrics to chart const chartMetrics = [ { key: "event_tps", label: "Event Throughput (TPS) — higher is better", file: "event_tps.tsv", ylabel: "TPS" }, @@ -161,7 +149,7 @@ for (const cm of chartMetrics) { e.servers["parrhesia-memory"]?.[cm.key] ?? "NaN", ]; for (const srv of presentBaselines) { - row.push(baselineAvg[srv]?.[cm.key] ?? "NaN"); + row.push(e.servers[srv]?.[cm.key] ?? "NaN"); } rows.push(row.join("\t")); } @@ -171,7 +159,7 @@ for (const cm of chartMetrics) { // Generate gnuplot plot commands (handles variable column counts) const serverLabels = ["parrhesia-pg", "parrhesia-memory"]; -for (const srv of presentBaselines) serverLabels.push(srv + " (avg)"); +for (const srv of presentBaselines) serverLabels.push(srv); const plotLines = []; for (const cm of chartMetrics) { @@ -219,7 +207,7 @@ const fs = require("node:fs"); const [, , jsonOut, readmePath] = process.argv; -const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8")); +const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8")); const readme = fs.readFileSync(readmePath, "utf8"); const pg = servers["parrhesia-pg"];