diff --git a/README.md b/README.md index 2cc074b..07534b2 100644 --- a/README.md +++ b/README.md @@ -550,16 +550,16 @@ mix bench Current comparison results from [BENCHMARK.md](./BENCHMARK.md): -| metric | parrhesia-pg | parrhesia-mem | nostr-rs-relay | mem/pg | nostr-rs/pg | -| --- | ---: | ---: | ---: | ---: | ---: | -| connect avg latency (ms) ↓ | 9.33 | 7.67 | 7.00 | **0.82x** | **0.75x** | -| connect max latency (ms) ↓ | 12.33 | 9.67 | 10.33 | **0.78x** | **0.84x** | -| echo throughput (TPS) ↑ | 64030.33 | 93656.33 | 140767.00 | **1.46x** | **2.20x** | -| echo throughput (MiB/s) ↑ | 35.07 | 51.27 | 77.07 | **1.46x** | **2.20x** | -| event throughput (TPS) ↑ | 5015.33 | 1505.33 | 2293.67 | 0.30x | 0.46x | -| event throughput (MiB/s) ↑ | 3.40 | 1.00 | 1.50 | 0.29x | 0.44x | -| req throughput (TPS) ↑ | 6416.33 | 14566.67 | 3035.67 | **2.27x** | 0.47x | -| req throughput (MiB/s) ↑ | 42.43 | 94.23 | 19.23 | **2.22x** | 0.45x | +| metric | parrhesia-pg | parrhesia-mem | strfry | nostr-rs-relay | mem/pg | strfry/pg | nostr-rs/pg | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| connect avg latency (ms) ↓ | 26.67 | 14.67 | 3.00 | 2.00 | **0.55x** | **0.11x** | **0.07x** | +| connect max latency (ms) ↓ | 45.33 | 24.33 | 4.67 | 3.33 | **0.54x** | **0.10x** | **0.07x** | +| echo throughput (TPS) ↑ | 68100.33 | 55978.00 | 67718.33 | 166178.00 | 0.82x | 0.99x | **2.44x** | +| echo throughput (MiB/s) ↑ | 37.23 | 30.63 | 37.87 | 91.03 | 0.82x | **1.02x** | **2.44x** | +| event throughput (TPS) ↑ | 1647.33 | 882.00 | 3548.33 | 787.00 | 0.54x | **2.15x** | 0.48x | +| event throughput (MiB/s) ↑ | 1.07 | 0.57 | 2.30 | 0.50 | 0.53x | **2.16x** | 0.47x | +| req throughput (TPS) ↑ | 3576.67 | 6888.00 | 1808.00 | 860.67 | **1.93x** | 0.51x | 0.24x | +| req throughput (MiB/s) ↑ | 18.83 | 36.07 | 11.70 | 2.40 | **1.92x** | 0.62x | 0.13x | Higher is better for `↑` metrics. Lower is better for `↓` metrics. diff --git a/bench/chart.svg b/bench/chart.svg index 8576853..f993cdf 100644 --- a/bench/chart.svg +++ b/bench/chart.svg @@ -51,11 +51,37 @@ - + - + + 500 + + + + + + + + + + + + + 1000 + + + + + + + + + + + + 1500 @@ -64,11 +90,11 @@ - + - + 2000 @@ -77,11 +103,11 @@ - + - + 2500 @@ -90,11 +116,11 @@ - + - + 3000 @@ -103,11 +129,11 @@ - + - + 3500 @@ -116,62 +142,23 @@ - + - + 4000 - - - - - - - - - - 4500 - - - - - - - - - - - - - 5000 - - - - - - - - - - - - - 5500 - - - - - v0.5.0 + + v0.6.0 - + parrhesia-pg @@ -183,7 +170,7 @@ - + @@ -194,38 +181,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + TPS - + Event Throughput (TPS) — higher is better @@ -236,11 +234,37 @@ - + - + + 0 + + + + + + + + + + + + + 1000 + + + + + + + + + + + + 2000 @@ -249,11 +273,24 @@ - + - + + 3000 + + + + + + + + + + + + 4000 @@ -262,11 +299,24 @@ - + - + + 5000 + + + + + + + + + + + + 6000 @@ -275,75 +325,23 @@ - + - - 8000 + + 7000 - - - - - - - - - - 10000 - - - - - - - - - - - - - 12000 - - - - - - - - - - - - - 14000 - - - - - - - - - - - - - 16000 - - - - - v0.5.0 + + v0.6.0 - + parrhesia-pg @@ -355,7 +353,7 @@ - + @@ -366,38 +364,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + TPS - + Req Throughput (TPS) — higher is better @@ -408,11 +417,24 @@ - + - + + 40000 + + + + + + + + + + + + 60000 @@ -421,24 +443,11 @@ - + - - 70000 - - - - - - - - - - - - + 80000 @@ -447,24 +456,11 @@ - + - - 90000 - - - - - - - - - - - - + 100000 @@ -473,24 +469,11 @@ - + - - 110000 - - - - - - - - - - - - + 120000 @@ -499,24 +482,11 @@ - + - - 130000 - - - - - - - - - - - - + 140000 @@ -525,23 +495,36 @@ - + - - 150000 + + 160000 - - v0.5.0 + + + + + + + + + + 180000 + + + + + v0.6.0 - + parrhesia-pg @@ -553,7 +536,7 @@ - + @@ -564,38 +547,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + TPS - + Echo Throughput (TPS) — higher is better @@ -606,12 +600,12 @@ - + - - 7 + + 0 @@ -619,12 +613,12 @@ - + - - 7.5 + + 5 @@ -632,12 +626,12 @@ - + - - 8 + + 10 @@ -645,12 +639,12 @@ - + - - 8.5 + + 15 @@ -658,12 +652,12 @@ - + - - 9 + + 20 @@ -671,23 +665,36 @@ - + - - 9.5 + + 25 - - v0.5.0 + + + + + + + + + + 30 + + + + + v0.6.0 - + parrhesia-pg @@ -699,7 +706,7 @@ - + @@ -710,38 +717,49 @@ - + - nostr-rs-relay (avg) + strfry - nostr-rs-relay (avg) + strfry - + - + nostr-rs-relay + + + nostr-rs-relay + + + + + + + + - + ms - + Connect Avg Latency (ms) — lower is better diff --git a/bench/history.jsonl b/bench/history.jsonl new file mode 100644 index 0000000..738d9b1 --- /dev/null +++ b/bench/history.jsonl @@ -0,0 +1 @@ +{"timestamp":"2026-03-18T21:35:03Z","machine_id":"agent","git_tag":"v0.6.0","git_commit":"7b337d9","runs":3,"versions":{"parrhesia":"0.6.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":26.666666666666668,"connect_max_ms":45.333333333333336,"echo_tps":68100.33333333333,"echo_mibs":37.233333333333334,"event_tps":1647.3333333333333,"event_mibs":1.0666666666666667,"req_tps":3576.6666666666665,"req_mibs":18.833333333333332},"parrhesia-memory":{"connect_avg_ms":14.666666666666666,"connect_max_ms":24.333333333333332,"echo_tps":55978,"echo_mibs":30.633333333333336,"event_tps":882,"event_mibs":0.5666666666666668,"req_tps":6888,"req_mibs":36.06666666666666},"strfry":{"connect_avg_ms":3,"connect_max_ms":4.666666666666667,"echo_tps":67718.33333333333,"echo_mibs":37.86666666666667,"event_tps":3548.3333333333335,"event_mibs":2.3,"req_tps":1808,"req_mibs":11.699999999999998},"nostr-rs-relay":{"connect_avg_ms":2,"connect_max_ms":3.3333333333333335,"echo_tps":166178,"echo_mibs":91.03333333333335,"event_tps":787,"event_mibs":0.5,"req_tps":860.6666666666666,"req_mibs":2.4}}}