diff --git a/README.md b/README.md index 07534b2..9b6509f 100644 --- a/README.md +++ b/README.md @@ -548,23 +548,25 @@ Run it with: mix bench ``` -Current comparison results from [BENCHMARK.md](./BENCHMARK.md): +Current comparison results: | metric | parrhesia-pg | parrhesia-mem | strfry | nostr-rs-relay | mem/pg | strfry/pg | nostr-rs/pg | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| connect avg latency (ms) ↓ | 26.67 | 14.67 | 3.00 | 2.00 | **0.55x** | **0.11x** | **0.07x** | -| connect max latency (ms) ↓ | 45.33 | 24.33 | 4.67 | 3.33 | **0.54x** | **0.10x** | **0.07x** | -| echo throughput (TPS) ↑ | 68100.33 | 55978.00 | 67718.33 | 166178.00 | 0.82x | 0.99x | **2.44x** | -| echo throughput (MiB/s) ↑ | 37.23 | 30.63 | 37.87 | 91.03 | 0.82x | **1.02x** | **2.44x** | -| event throughput (TPS) ↑ | 1647.33 | 882.00 | 3548.33 | 787.00 | 0.54x | **2.15x** | 0.48x | -| event throughput (MiB/s) ↑ | 1.07 | 0.57 | 2.30 | 0.50 | 0.53x | **2.16x** | 0.47x | -| req throughput (TPS) ↑ | 3576.67 | 6888.00 | 1808.00 | 860.67 | **1.93x** | 0.51x | 0.24x | -| req throughput (MiB/s) ↑ | 18.83 | 36.07 | 11.70 | 2.40 | **1.92x** | 0.62x | 0.13x | +| connect avg latency (ms) ↓ | 34.67 | 43.33 | 2.67 | 2.67 | 1.25x | **0.08x** | **0.08x** | +| connect max latency (ms) ↓ | 61.67 | 74.67 | 4.67 | 4.00 | 1.21x | **0.08x** | **0.06x** | +| echo throughput (TPS) ↑ | 72441.00 | 62704.67 | 61189.33 | 152654.33 | 0.87x | 0.84x | **2.11x** | +| echo throughput (MiB/s) ↑ | 39.67 | 34.30 | 34.20 | 83.63 | 0.86x | 0.86x | **2.11x** | +| event throughput (TPS) ↑ | 1897.33 | 1370.00 | 3426.67 | 772.67 | 0.72x | **1.81x** | 0.41x | +| event throughput (MiB/s) ↑ | 1.23 | 0.87 | 2.20 | 0.50 | 0.70x | **1.78x** | 0.41x | +| req throughput (TPS) ↑ | 13.33 | 47.00 | 1811.33 | 878.33 | **3.52x** | **135.85x** | **65.88x** | +| req throughput (MiB/s) ↑ | 0.03 | 0.17 | 11.77 | 2.40 | **5.00x** | **353.00x** | **72.00x** | Higher is better for `↑` metrics. Lower is better for `↓` metrics. (Results from a Linux container on a 6-core Intel i5-8400T with NVMe drive, PostgreSQL 18) +![Benchmark chart](./bench/chart.svg) + --- ## Development quality checks diff --git a/bench/chart.gnuplot b/bench/chart.gnuplot index 6f8bc38..ab6ce2f 100644 --- a/bench/chart.gnuplot +++ b/bench/chart.gnuplot @@ -7,7 +7,7 @@ # fragment generated by the data-prep step that defines the actual plot # directives (handling variable server columns). -set terminal svg enhanced size 1200,900 font "sans,11" +set terminal svg enhanced size 1200,900 font "sans-serif,11" background "#f3f4f6" set output output_file set style data linespoints diff --git a/bench/chart.svg b/bench/chart.svg index 81df9e4..72d210d 100644 --- a/bench/chart.svg +++ b/bench/chart.svg @@ -11,7 +11,7 @@ - + @@ -31,7 +31,7 @@ - + @@ -39,9 +39,9 @@ - - - Parrhesia Relay Benchmark History + + + Parrhesia Relay Benchmark History @@ -55,8 +55,8 @@ - - 500 + + 500 @@ -68,8 +68,8 @@ - - 1000 + + 1000 @@ -81,8 +81,8 @@ - - 1500 + + 1500 @@ -94,8 +94,8 @@ - - 2000 + + 2000 @@ -107,8 +107,8 @@ - - 2500 + + 2500 @@ -120,8 +120,8 @@ - - 3000 + + 3000 @@ -133,8 +133,8 @@ - - 3500 + + 3500 @@ -146,33 +146,33 @@ - - 4000 + + 4000 - - v0.2.0 + + v0.2.0 - - v0.3.0 + + v0.3.0 - - v0.4.0 + + v0.4.0 - - v0.5.0 + + v0.5.0 - - v0.6.0 + + v0.6.0 @@ -182,11 +182,11 @@ parrhesia-pg - + - - parrhesia-pg + + parrhesia-pg @@ -200,8 +200,8 @@ parrhesia-memory - - parrhesia-memory + + parrhesia-memory @@ -214,11 +214,11 @@ strfry - + - - strfry + + strfry @@ -232,8 +232,8 @@ nostr-rs-relay - - nostr-rs-relay + + nostr-rs-relay @@ -245,22 +245,22 @@ - + - - TPS + + TPS - - Event Throughput (TPS) — higher is better + + Event Throughput (TPS) — higher is better @@ -274,8 +274,8 @@ - - 0 + + 0 @@ -287,8 +287,8 @@ - - 1000 + + 1000 @@ -300,8 +300,8 @@ - - 2000 + + 2000 @@ -313,8 +313,8 @@ - - 3000 + + 3000 @@ -326,8 +326,8 @@ - - 4000 + + 4000 @@ -339,8 +339,8 @@ - - 5000 + + 5000 @@ -352,8 +352,8 @@ - - 6000 + + 6000 @@ -365,33 +365,33 @@ - - 7000 + + 7000 - - v0.2.0 + + v0.2.0 - - v0.3.0 + + v0.3.0 - - v0.4.0 + + v0.4.0 - - v0.5.0 + + v0.5.0 - - v0.6.0 + + v0.6.0 @@ -401,11 +401,11 @@ parrhesia-pg - + - - parrhesia-pg + + parrhesia-pg @@ -419,8 +419,8 @@ parrhesia-memory - - parrhesia-memory + + parrhesia-memory @@ -433,11 +433,11 @@ strfry - + - - strfry + + strfry @@ -451,8 +451,8 @@ nostr-rs-relay - - nostr-rs-relay + + nostr-rs-relay @@ -464,22 +464,22 @@ - + - - TPS + + TPS - - Req Throughput (TPS) — higher is better + + Req Throughput (TPS) — higher is better @@ -493,8 +493,8 @@ - - 40000 + + 40000 @@ -506,8 +506,8 @@ - - 60000 + + 60000 @@ -519,8 +519,8 @@ - - 80000 + + 80000 @@ -532,8 +532,8 @@ - - 100000 + + 100000 @@ -545,8 +545,8 @@ - - 120000 + + 120000 @@ -558,8 +558,8 @@ - - 140000 + + 140000 @@ -571,8 +571,8 @@ - - 160000 + + 160000 @@ -584,33 +584,33 @@ - - 180000 + + 180000 - - v0.2.0 + + v0.2.0 - - v0.3.0 + + v0.3.0 - - v0.4.0 + + v0.4.0 - - v0.5.0 + + v0.5.0 - - v0.6.0 + + v0.6.0 @@ -620,11 +620,11 @@ parrhesia-pg - + - - parrhesia-pg + + parrhesia-pg @@ -638,8 +638,8 @@ parrhesia-memory - - parrhesia-memory + + parrhesia-memory @@ -652,11 +652,11 @@ strfry - + - - strfry + + strfry @@ -670,8 +670,8 @@ nostr-rs-relay - - nostr-rs-relay + + nostr-rs-relay @@ -683,22 +683,22 @@ - + - - TPS + + TPS - - Echo Throughput (TPS) — higher is better + + Echo Throughput (TPS) — higher is better @@ -712,8 +712,8 @@ - - 0 + + 0 @@ -725,8 +725,8 @@ - - 5 + + 5 @@ -738,8 +738,8 @@ - - 10 + + 10 @@ -751,8 +751,8 @@ - - 15 + + 15 @@ -764,8 +764,8 @@ - - 20 + + 20 @@ -777,8 +777,8 @@ - - 25 + + 25 @@ -790,8 +790,8 @@ - - 30 + + 30 @@ -803,8 +803,8 @@ - - 35 + + 35 @@ -816,8 +816,8 @@ - - 40 + + 40 @@ -829,33 +829,33 @@ - - 45 + + 45 - - v0.2.0 + + v0.2.0 - - v0.3.0 + + v0.3.0 - - v0.4.0 + + v0.4.0 - - v0.5.0 + + v0.5.0 - - v0.6.0 + + v0.6.0 @@ -865,11 +865,11 @@ parrhesia-pg - + - - parrhesia-pg + + parrhesia-pg @@ -883,8 +883,8 @@ parrhesia-memory - - parrhesia-memory + + parrhesia-memory @@ -897,11 +897,11 @@ strfry - + - - strfry + + strfry @@ -915,8 +915,8 @@ nostr-rs-relay - - nostr-rs-relay + + nostr-rs-relay @@ -928,22 +928,22 @@ - + - - ms + + ms - - Connect Avg Latency (ms) — lower is better + + Connect Avg Latency (ms) — lower is better