diff --git a/README.md b/README.md
index 2cc074b..07534b2 100644
--- a/README.md
+++ b/README.md
@@ -550,16 +550,16 @@ mix bench
Current comparison results from [BENCHMARK.md](./BENCHMARK.md):
-| metric | parrhesia-pg | parrhesia-mem | nostr-rs-relay | mem/pg | nostr-rs/pg |
-| --- | ---: | ---: | ---: | ---: | ---: |
-| connect avg latency (ms) ↓ | 9.33 | 7.67 | 7.00 | **0.82x** | **0.75x** |
-| connect max latency (ms) ↓ | 12.33 | 9.67 | 10.33 | **0.78x** | **0.84x** |
-| echo throughput (TPS) ↑ | 64030.33 | 93656.33 | 140767.00 | **1.46x** | **2.20x** |
-| echo throughput (MiB/s) ↑ | 35.07 | 51.27 | 77.07 | **1.46x** | **2.20x** |
-| event throughput (TPS) ↑ | 5015.33 | 1505.33 | 2293.67 | 0.30x | 0.46x |
-| event throughput (MiB/s) ↑ | 3.40 | 1.00 | 1.50 | 0.29x | 0.44x |
-| req throughput (TPS) ↑ | 6416.33 | 14566.67 | 3035.67 | **2.27x** | 0.47x |
-| req throughput (MiB/s) ↑ | 42.43 | 94.23 | 19.23 | **2.22x** | 0.45x |
+| metric | parrhesia-pg | parrhesia-mem | strfry | nostr-rs-relay | mem/pg | strfry/pg | nostr-rs/pg |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| connect avg latency (ms) ↓ | 26.67 | 14.67 | 3.00 | 2.00 | **0.55x** | **0.11x** | **0.07x** |
+| connect max latency (ms) ↓ | 45.33 | 24.33 | 4.67 | 3.33 | **0.54x** | **0.10x** | **0.07x** |
+| echo throughput (TPS) ↑ | 68100.33 | 55978.00 | 67718.33 | 166178.00 | 0.82x | 0.99x | **2.44x** |
+| echo throughput (MiB/s) ↑ | 37.23 | 30.63 | 37.87 | 91.03 | 0.82x | **1.02x** | **2.44x** |
+| event throughput (TPS) ↑ | 1647.33 | 882.00 | 3548.33 | 787.00 | 0.54x | **2.15x** | 0.48x |
+| event throughput (MiB/s) ↑ | 1.07 | 0.57 | 2.30 | 0.50 | 0.53x | **2.16x** | 0.47x |
+| req throughput (TPS) ↑ | 3576.67 | 6888.00 | 1808.00 | 860.67 | **1.93x** | 0.51x | 0.24x |
+| req throughput (MiB/s) ↑ | 18.83 | 36.07 | 11.70 | 2.40 | **1.92x** | 0.62x | 0.13x |
Higher is better for `↑` metrics. Lower is better for `↓` metrics.
diff --git a/bench/chart.svg b/bench/chart.svg
index 8576853..f993cdf 100644
--- a/bench/chart.svg
+++ b/bench/chart.svg
@@ -51,11 +51,37 @@
-
+
-
+
+ 500
+
+
+
+
+
+
+
+
+
+
+
+
+ 1000
+
+
+
+
+
+
+
+
+
+
+
+
1500
@@ -64,11 +90,11 @@
-
+
-
+
2000
@@ -77,11 +103,11 @@
-
+
-
+
2500
@@ -90,11 +116,11 @@
-
+
-
+
3000
@@ -103,11 +129,11 @@
-
+
-
+
3500
@@ -116,62 +142,23 @@
-
+
-
+
4000
-
-
-
-
-
-
-
-
-
- 4500
-
-
-
-
-
-
-
-
-
-
-
-
- 5000
-
-
-
-
-
-
-
-
-
-
-
-
- 5500
-
-
-
-
- v0.5.0
+
+ v0.6.0
-
+
parrhesia-pg
@@ -183,7 +170,7 @@
-
+
@@ -194,38 +181,49 @@
-
+
- nostr-rs-relay (avg)
+ strfry
- nostr-rs-relay (avg)
+ strfry
-
+
-
+ nostr-rs-relay
+
+
+ nostr-rs-relay
+
+
+
+
+
+
+
+
-
+
TPS
-
+
Event Throughput (TPS) — higher is better
@@ -236,11 +234,37 @@
-
+
-
+
+ 0
+
+
+
+
+
+
+
+
+
+
+
+
+ 1000
+
+
+
+
+
+
+
+
+
+
+
+
2000
@@ -249,11 +273,24 @@
-
+
-
+
+ 3000
+
+
+
+
+
+
+
+
+
+
+
+
4000
@@ -262,11 +299,24 @@
-
+
-
+
+ 5000
+
+
+
+
+
+
+
+
+
+
+
+
6000
@@ -275,75 +325,23 @@
-
+
-
- 8000
+
+ 7000
-
-
-
-
-
-
-
-
-
- 10000
-
-
-
-
-
-
-
-
-
-
-
-
- 12000
-
-
-
-
-
-
-
-
-
-
-
-
- 14000
-
-
-
-
-
-
-
-
-
-
-
-
- 16000
-
-
-
-
- v0.5.0
+
+ v0.6.0
-
+
parrhesia-pg
@@ -355,7 +353,7 @@
-
+
@@ -366,38 +364,49 @@
-
+
- nostr-rs-relay (avg)
+ strfry
- nostr-rs-relay (avg)
+ strfry
-
+
-
+ nostr-rs-relay
+
+
+ nostr-rs-relay
+
+
+
+
+
+
+
+
-
+
TPS
-
+
Req Throughput (TPS) — higher is better
@@ -408,11 +417,24 @@
-
+
-
+
+ 40000
+
+
+
+
+
+
+
+
+
+
+
+
60000
@@ -421,24 +443,11 @@
-
+
-
- 70000
-
-
-
-
-
-
-
-
-
-
-
-
+
80000
@@ -447,24 +456,11 @@
-
+
-
- 90000
-
-
-
-
-
-
-
-
-
-
-
-
+
100000
@@ -473,24 +469,11 @@
-
+
-
- 110000
-
-
-
-
-
-
-
-
-
-
-
-
+
120000
@@ -499,24 +482,11 @@
-
+
-
- 130000
-
-
-
-
-
-
-
-
-
-
-
-
+
140000
@@ -525,23 +495,36 @@
-
+
-
- 150000
+
+ 160000
-
- v0.5.0
+
+
+
+
+
+
+
+
+
+ 180000
+
+
+
+
+ v0.6.0
-
+
parrhesia-pg
@@ -553,7 +536,7 @@
-
+
@@ -564,38 +547,49 @@
-
+
- nostr-rs-relay (avg)
+ strfry
- nostr-rs-relay (avg)
+ strfry
-
+
-
+ nostr-rs-relay
+
+
+ nostr-rs-relay
+
+
+
+
+
+
+
+
-
+
TPS
-
+
Echo Throughput (TPS) — higher is better
@@ -606,12 +600,12 @@
-
+
-
- 7
+
+ 0
@@ -619,12 +613,12 @@
-
+
-
- 7.5
+
+ 5
@@ -632,12 +626,12 @@
-
+
-
- 8
+
+ 10
@@ -645,12 +639,12 @@
-
+
-
- 8.5
+
+ 15
@@ -658,12 +652,12 @@
-
+
-
- 9
+
+ 20
@@ -671,23 +665,36 @@
-
+
-
- 9.5
+
+ 25
-
- v0.5.0
+
+
+
+
+
+
+
+
+
+ 30
+
+
+
+
+ v0.6.0
-
+
parrhesia-pg
@@ -699,7 +706,7 @@
-
+
@@ -710,38 +717,49 @@
-
+
- nostr-rs-relay (avg)
+ strfry
- nostr-rs-relay (avg)
+ strfry
-
+
-
+ nostr-rs-relay
+
+
+ nostr-rs-relay
+
+
+
+
+
+
+
+
-
+
ms
-
+
Connect Avg Latency (ms) — lower is better
diff --git a/bench/history.jsonl b/bench/history.jsonl
index 98dac89..738d9b1 100644
--- a/bench/history.jsonl
+++ b/bench/history.jsonl
@@ -1 +1 @@
-{"timestamp":"2026-03-18T20:13:21Z","machine_id":"squirrel","git_tag":"v0.5.0","git_commit":"970cee2","runs":3,"servers":{"parrhesia-pg":{"connect_avg_ms":9.333333333333334,"connect_max_ms":12.333333333333334,"echo_tps":64030.333333333336,"echo_mibs":35.06666666666666,"event_tps":5015.333333333333,"event_mibs":3.4,"req_tps":6416.333333333333,"req_mibs":42.43333333333334},"parrhesia-memory":{"connect_avg_ms":7.666666666666667,"connect_max_ms":9.666666666666666,"echo_tps":93656.33333333333,"echo_mibs":51.26666666666667,"event_tps":1505.3333333333333,"event_mibs":1,"req_tps":14566.666666666666,"req_mibs":94.23333333333335},"nostr-rs-relay":{"connect_avg_ms":7,"connect_max_ms":10.333333333333334,"echo_tps":140767,"echo_mibs":77.06666666666666,"event_tps":2293.6666666666665,"event_mibs":1.5,"req_tps":3035.6666666666665,"req_mibs":19.23333333333333}}}
+{"timestamp":"2026-03-18T21:35:03Z","machine_id":"agent","git_tag":"v0.6.0","git_commit":"7b337d9","runs":3,"versions":{"parrhesia":"0.6.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":26.666666666666668,"connect_max_ms":45.333333333333336,"echo_tps":68100.33333333333,"echo_mibs":37.233333333333334,"event_tps":1647.3333333333333,"event_mibs":1.0666666666666667,"req_tps":3576.6666666666665,"req_mibs":18.833333333333332},"parrhesia-memory":{"connect_avg_ms":14.666666666666666,"connect_max_ms":24.333333333333332,"echo_tps":55978,"echo_mibs":30.633333333333336,"event_tps":882,"event_mibs":0.5666666666666668,"req_tps":6888,"req_mibs":36.06666666666666},"strfry":{"connect_avg_ms":3,"connect_max_ms":4.666666666666667,"echo_tps":67718.33333333333,"echo_mibs":37.86666666666667,"event_tps":3548.3333333333335,"event_mibs":2.3,"req_tps":1808,"req_mibs":11.699999999999998},"nostr-rs-relay":{"connect_avg_ms":2,"connect_max_ms":3.3333333333333335,"echo_tps":166178,"echo_mibs":91.03333333333335,"event_tps":787,"event_mibs":0.5,"req_tps":860.6666666666666,"req_mibs":2.4}}}
diff --git a/scripts/run_bench_compare.sh b/scripts/run_bench_compare.sh
index 094bef4..f0e15c9 100755
--- a/scripts/run_bench_compare.sh
+++ b/scripts/run_bench_compare.sh
@@ -298,7 +298,8 @@ for run in $(seq 1 "$RUNS"); do
done
-node - "$WORK_DIR" "$RUNS" "$HAS_STRFRY" "$HAS_NOSTR_RS" <<'NODE'
+node - "$WORK_DIR" "$RUNS" "$HAS_STRFRY" "$HAS_NOSTR_RS" \
+ "$PARRHESIA_VERSION" "$STRFRY_VERSION" "$NOSTR_RS_RELAY_VERSION" "$NOSTR_BENCH_VERSION" <<'NODE'
const fs = require("node:fs");
const path = require("node:path");
@@ -306,6 +307,10 @@ const workDir = process.argv[2];
const runs = Number(process.argv[3]);
const hasStrfry = process.argv[4] === "1";
const hasNostrRs = process.argv[5] === "1";
+const parrhesiaVersion = process.argv[6] || "";
+const strfryVersion = process.argv[7] || "";
+const nostrRsRelayVersion = process.argv[8] || "";
+const nostrBenchVersion = process.argv[9] || "";
function parseLog(filePath) {
const content = fs.readFileSync(filePath, "utf8");
@@ -502,6 +507,12 @@ if (process.env.BENCH_JSON_OUT) {
};
}
+ const versions = { parrhesia: parrhesiaVersion };
+ if (hasStrfry && strfryVersion) versions.strfry = strfryVersion;
+ if (hasNostrRs && nostrRsRelayVersion) versions["nostr-rs-relay"] = nostrRsRelayVersion;
+ if (nostrBenchVersion) versions["nostr-bench"] = nostrBenchVersion;
+ jsonSummary.versions = versions;
+
fs.writeFileSync(
process.env.BENCH_JSON_OUT,
JSON.stringify(jsonSummary, null, 2) + "\n",
diff --git a/scripts/run_bench_update.sh b/scripts/run_bench_update.sh
index 7fa06d3..06e011f 100755
--- a/scripts/run_bench_update.sh
+++ b/scripts/run_bench_update.sh
@@ -70,7 +70,7 @@ const fs = require("node:fs");
const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv;
-const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
+const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
const entry = {
timestamp,
@@ -78,6 +78,7 @@ const entry = {
git_tag: gitTag,
git_commit: gitCommit,
runs: Number(runsStr),
+ versions: versions || {},
servers,
};
@@ -127,19 +128,6 @@ const presentBaselines = baselineServerNames.filter(srv =>
deduped.some(e => e.servers[srv])
);
-// Compute averages for baseline servers (constant horizontal lines)
-const baselineAvg = {};
-for (const srv of presentBaselines) {
- const vals = deduped.filter(e => e.servers[srv]).map(e => e.servers[srv]);
- baselineAvg[srv] = {};
- for (const metric of Object.keys(vals[0])) {
- const valid = vals.map(v => v[metric]).filter(Number.isFinite);
- baselineAvg[srv][metric] = valid.length > 0
- ? valid.reduce((a, b) => a + b, 0) / valid.length
- : NaN;
- }
-}
-
// Metrics to chart
const chartMetrics = [
{ key: "event_tps", label: "Event Throughput (TPS) — higher is better", file: "event_tps.tsv", ylabel: "TPS" },
@@ -161,7 +149,7 @@ for (const cm of chartMetrics) {
e.servers["parrhesia-memory"]?.[cm.key] ?? "NaN",
];
for (const srv of presentBaselines) {
- row.push(baselineAvg[srv]?.[cm.key] ?? "NaN");
+ row.push(e.servers[srv]?.[cm.key] ?? "NaN");
}
rows.push(row.join("\t"));
}
@@ -171,7 +159,7 @@ for (const cm of chartMetrics) {
// Generate gnuplot plot commands (handles variable column counts)
const serverLabels = ["parrhesia-pg", "parrhesia-memory"];
-for (const srv of presentBaselines) serverLabels.push(srv + " (avg)");
+for (const srv of presentBaselines) serverLabels.push(srv);
const plotLines = [];
for (const cm of chartMetrics) {
@@ -219,7 +207,7 @@ const fs = require("node:fs");
const [, , jsonOut, readmePath] = process.argv;
-const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
+const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
const readme = fs.readFileSync(readmePath, "utf8");
const pg = servers["parrhesia-pg"];