Benchmark capture and plot
Some checks failed
CI / Test (OTP 27.2 / Elixir 1.18.2) (push) Failing after 0s
CI / Test (OTP 28.4 / Elixir 1.19.4 + Marmot E2E) (push) Failing after 0s

This commit is contained in:
2026-03-18 21:23:23 +01:00
parent 970cee2c0e
commit fc3d121599
9 changed files with 1157 additions and 43 deletions

View File

@@ -1,33 +0,0 @@
Running 2 comparison run(s)...
Versions:
parrhesia 0.4.0
strfry 1.0.4 (nixpkgs)
nostr-rs-relay 0.9.0
nostr-bench 0.4.0
[run 1/2] Parrhesia
[run 1/2] strfry
[run 1/2] nostr-rs-relay
[run 2/2] Parrhesia
[run 2/2] strfry
[run 2/2] nostr-rs-relay
=== Bench comparison (averages) ===
metric parrhesia strfry nostr-rs-relay strfry/parrhesia nostr-rs/parrhesia
-------------------------- --------- -------- -------------- ---------------- ------------------
connect avg latency (ms) ↓ 10.50 4.00 3.00 0.38x 0.29x
connect max latency (ms) ↓ 19.50 7.50 4.00 0.38x 0.21x
echo throughput (TPS) ↑ 78520.00 60353.00 164420.50 0.77x 2.09x
echo throughput (MiB/s) ↑ 43.00 33.75 90.05 0.78x 2.09x
event throughput (TPS) ↑ 1919.50 3520.50 781.00 1.83x 0.41x
event throughput (MiB/s) ↑ 1.25 2.25 0.50 1.80x 0.40x
req throughput (TPS) ↑ 4608.50 1809.50 875.50 0.39x 0.19x
req throughput (MiB/s) ↑ 26.20 11.75 2.40 0.45x 0.09x
Legend: ↑ higher is better, ↓ lower is better.
Ratio columns are server/parrhesia (for ↓ metrics, <1.00x means that server is faster).
Run details:
run 1: parrhesia(echo_tps=78892, event_tps=1955, req_tps=4671, connect_avg_ms=10) | strfry(echo_tps=59132, event_tps=3462, req_tps=1806, connect_avg_ms=4) | nostr-rs-relay(echo_tps=159714, event_tps=785, req_tps=873, connect_avg_ms=3)
run 2: parrhesia(echo_tps=78148, event_tps=1884, req_tps=4546, connect_avg_ms=11) | strfry(echo_tps=61574, event_tps=3579, req_tps=1813, connect_avg_ms=4) | nostr-rs-relay(echo_tps=169127, event_tps=777, req_tps=878, connect_avg_ms=3)

View File

@@ -550,16 +550,16 @@ mix bench
Current comparison results from [BENCHMARK.md](./BENCHMARK.md):
| metric | parrhesia | strfry | nostr-rs-relay | strfry/parrhesia | nostr-rs/parrhesia |
| metric | parrhesia-pg | parrhesia-mem | nostr-rs-relay | mem/pg | nostr-rs/pg |
| --- | ---: | ---: | ---: | ---: | ---: |
| connect avg latency (ms) ↓ | 13.50 | 3.00 | 2.00 | **0.22x** | **0.15x** |
| connect max latency (ms) ↓ | 22.50 | 5.50 | 3.00 | **0.24x** | **0.13x** |
| echo throughput (TPS) ↑ | 80385.00 | 61673.00 | 164516.00 | 0.77x | **2.05x** |
| echo throughput (MiB/s) ↑ | 44.00 | 34.45 | 90.10 | 0.78x | **2.05x** |
| event throughput (TPS) ↑ | 2000.00 | 3404.50 | 788.00 | **1.70x** | 0.39x |
| event throughput (MiB/s) ↑ | 1.30 | 2.20 | 0.50 | **1.69x** | 0.38x |
| req throughput (TPS) ↑ | 3664.00 | 1808.50 | 877.50 | 0.49x | 0.24x |
| req throughput (MiB/s) ↑ | 20.75 | 11.75 | 2.45 | 0.57x | 0.12x |
| connect avg latency (ms) ↓ | 9.33 | 7.67 | 7.00 | **0.82x** | **0.75x** |
| connect max latency (ms) ↓ | 12.33 | 9.67 | 10.33 | **0.78x** | **0.84x** |
| echo throughput (TPS) ↑ | 64030.33 | 93656.33 | 140767.00 | **1.46x** | **2.20x** |
| echo throughput (MiB/s) ↑ | 35.07 | 51.27 | 77.07 | **1.46x** | **2.20x** |
| event throughput (TPS) ↑ | 5015.33 | 1505.33 | 2293.67 | 0.30x | 0.46x |
| event throughput (MiB/s) ↑ | 3.40 | 1.00 | 1.50 | 0.29x | 0.44x |
| req throughput (TPS) ↑ | 6416.33 | 14566.67 | 3035.67 | **2.27x** | 0.47x |
| req throughput (MiB/s) ↑ | 42.43 | 94.23 | 19.23 | **2.22x** | 0.45x |
Higher is better for `↑` metrics. Lower is better for `↓` metrics.

31
bench/chart.gnuplot Normal file
View File

@@ -0,0 +1,31 @@
# bench/chart.gnuplot — multi-panel SVG showing relay performance over git tags.
#
# Invoked by scripts/run_bench_update.sh with:
# gnuplot -e "data_dir='...'" -e "output_file='...'" bench/chart.gnuplot
#
# The data_dir contains per-metric TSV files and a plot_commands.gnuplot
# fragment generated by the data-prep step that defines the actual plot
# directives (handling variable server columns).
set terminal svg enhanced size 1200,900 font "sans,11"
set output output_file
set style data linespoints
set key outside right top
set grid ytics
set xtics rotate by -30
set datafile separator "\t"
# parrhesia-pg: blue solid, parrhesia-memory: green solid
# strfry: orange dashed, nostr-rs-relay: red dashed
set linetype 1 lc rgb "#2563eb" lw 2 pt 7 ps 1.0
set linetype 2 lc rgb "#16a34a" lw 2 pt 9 ps 1.0
set linetype 3 lc rgb "#ea580c" lw 1.5 pt 5 ps 0.8 dt 2
set linetype 4 lc rgb "#dc2626" lw 1.5 pt 4 ps 0.8 dt 2
set multiplot layout 2,2 title "Parrhesia Relay Benchmark History" font ",14"
# Load dynamically generated plot commands (handles variable column counts)
load data_dir."/plot_commands.gnuplot"
unset multiplot

752
bench/chart.svg Normal file
View File

@@ -0,0 +1,752 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<svg
width="1200" height="900"
viewBox="0 0 1200 900"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
>
<title>Gnuplot</title>
<desc>Produced by GNUPLOT 6.0 patchlevel 4 </desc>
<g id="gnuplot_canvas">
<rect x="0" y="0" width="1200" height="900" fill="none"/>
<defs>
<circle id='gpDot' r='0.5' stroke-width='0.5' stroke='currentColor'/>
<path id='gpPt0' stroke-width='0.242' stroke='currentColor' d='M-1,0 h2 M0,-1 v2'/>
<path id='gpPt1' stroke-width='0.242' stroke='currentColor' d='M-1,-1 L1,1 M1,-1 L-1,1'/>
<path id='gpPt2' stroke-width='0.242' stroke='currentColor' d='M-1,0 L1,0 M0,-1 L0,1 M-1,-1 L1,1 M-1,1 L1,-1'/>
<rect id='gpPt3' stroke-width='0.242' stroke='currentColor' x='-1' y='-1' width='2' height='2'/>
<rect id='gpPt4' stroke-width='0.242' stroke='currentColor' fill='currentColor' x='-1' y='-1' width='2' height='2'/>
<circle id='gpPt5' stroke-width='0.242' stroke='currentColor' cx='0' cy='0' r='1'/>
<use xlink:href='#gpPt5' id='gpPt6' fill='currentColor' stroke='none'/>
<path id='gpPt7' stroke-width='0.242' stroke='currentColor' d='M0,-1.33 L-1.33,0.67 L1.33,0.67 z'/>
<use xlink:href='#gpPt7' id='gpPt8' fill='currentColor' stroke='none'/>
<use xlink:href='#gpPt7' id='gpPt9' stroke='currentColor' transform='rotate(180)'/>
<use xlink:href='#gpPt9' id='gpPt10' fill='currentColor' stroke='none'/>
<use xlink:href='#gpPt3' id='gpPt11' stroke='currentColor' transform='rotate(45)'/>
<use xlink:href='#gpPt11' id='gpPt12' fill='currentColor' stroke='none'/>
<path id='gpPt13' stroke-width='0.242' stroke='currentColor' d='M0,1.330 L1.265,0.411 L0.782,-1.067 L-0.782,-1.076 L-1.265,0.411 z'/>
<use xlink:href='#gpPt13' id='gpPt14' fill='currentColor' stroke='none'/>
<filter id='textbox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
<feFlood flood-color='white' flood-opacity='1' result='bgnd'/>
<feComposite in='SourceGraphic' in2='bgnd' operator='atop'/>
</filter>
<filter id='greybox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
<feFlood flood-color='lightgrey' flood-opacity='1' result='grey'/>
<feComposite in='SourceGraphic' in2='grey' operator='atop'/>
</filter>
</defs>
<g fill="none" color="white" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(600.00,21.05)" stroke="none" fill="black" font-family="sans" font-size="14.00" text-anchor="middle">
<text><tspan font-family="sans" >Parrhesia Relay Benchmark History</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,420.94 L368.73,420.94 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,420.94 L82.42,420.94 M368.73,420.94 L360.48,420.94 '/> <g transform="translate(66.48,424.52)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 1500</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,377.14 L368.73,377.14 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,377.14 L82.42,377.14 M368.73,377.14 L360.48,377.14 '/> <g transform="translate(66.48,380.72)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 2000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,333.33 L368.73,333.33 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,333.33 L82.42,333.33 M368.73,333.33 L360.48,333.33 '/> <g transform="translate(66.48,336.91)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 2500</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,289.53 L368.73,289.53 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,289.53 L82.42,289.53 M368.73,289.53 L360.48,289.53 '/> <g transform="translate(66.48,293.11)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 3000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,245.72 L368.73,245.72 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,245.72 L82.42,245.72 M368.73,245.72 L360.48,245.72 '/> <g transform="translate(66.48,249.30)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 3500</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,201.92 L368.73,201.92 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,201.92 L82.42,201.92 M368.73,201.92 L360.48,201.92 '/> <g transform="translate(66.48,205.50)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 4000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,158.12 L368.73,158.12 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,158.12 L82.42,158.12 M368.73,158.12 L360.48,158.12 '/> <g transform="translate(66.48,161.70)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 4500</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,114.31 L368.73,114.31 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,114.31 L82.42,114.31 M368.73,114.31 L360.48,114.31 '/> <g transform="translate(66.48,117.89)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 5000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M74.17,70.51 L368.73,70.51 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,70.51 L82.42,70.51 M368.73,70.51 L360.48,70.51 '/> <g transform="translate(66.48,74.09)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 5500</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M221.45,420.94 L221.45,412.69 M221.45,70.51 L221.45,78.76 '/> <g transform="translate(219.66,431.73) rotate(30.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="start">
<text><tspan font-family="sans" >v0.5.0</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,70.51 L74.17,420.94 L368.73,420.94 L368.73,70.51 L74.17,70.51 Z '/></g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g id="gnuplot_plot_1a" ><title>parrhesia-pg</title>
<g fill="none" color="white" stroke="black" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(537.91,82.34)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-pg</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 37, 99, 235)' d='M545.60,78.76 L584.61,78.76 M221.45,112.97 '/> <use xlink:href='#gpPt6' transform='translate(221.45,112.97) scale(4.12)' color='rgb( 37, 99, 235)'/>
<use xlink:href='#gpPt6' transform='translate(565.10,78.76) scale(4.12)' color='rgb( 37, 99, 235)'/>
</g>
</g>
<g id="gnuplot_plot_2a" ><title>parrhesia-memory</title>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(537.91,98.84)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-memory</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 22, 163, 74)' d='M545.60,95.26 L584.61,95.26 M221.45,420.47 '/> <use xlink:href='#gpPt8' transform='translate(221.45,420.47) scale(4.12)' color='rgb( 22, 163, 74)'/>
<use xlink:href='#gpPt8' transform='translate(565.10,95.26) scale(4.12)' color='rgb( 22, 163, 74)'/>
</g>
</g>
<g id="gnuplot_plot_3a" ><title>nostr-rs-relay (avg)</title>
<g fill="none" color="white" stroke="rgb( 22, 163, 74)" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(537.91,115.34)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >nostr-rs-relay (avg)</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb(234, 88, 12)' stroke-dasharray='3.8,6.0' d='M545.60,111.76 L584.61,111.76 M221.45,351.41 '/> <use xlink:href='#gpPt4' transform='translate(221.45,351.41) scale(3.30)' color='rgb(234, 88, 12)'/>
<use xlink:href='#gpPt4' transform='translate(565.10,111.76) scale(3.30)' color='rgb(234, 88, 12)'/>
</g>
</g>
<g fill="none" color="white" stroke="rgb(234, 88, 12)" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M74.17,70.51 L74.17,420.94 L368.73,420.94 L368.73,70.51 L74.17,70.51 Z '/> <g transform="translate(17.58,245.73) rotate(270.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >TPS</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(221.45,49.34)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >Event Throughput (TPS) — higher is better</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,420.94 L968.73,420.94 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,420.94 L690.11,420.94 M968.73,420.94 L960.48,420.94 '/> <g transform="translate(674.17,424.52)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 2000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,370.88 L968.73,370.88 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,370.88 L690.11,370.88 M968.73,370.88 L960.48,370.88 '/> <g transform="translate(674.17,374.46)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 4000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,320.82 L968.73,320.82 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,320.82 L690.11,320.82 M968.73,320.82 L960.48,320.82 '/> <g transform="translate(674.17,324.40)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 6000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,270.76 L968.73,270.76 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,270.76 L690.11,270.76 M968.73,270.76 L960.48,270.76 '/> <g transform="translate(674.17,274.34)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 8000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,220.69 L968.73,220.69 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,220.69 L690.11,220.69 M968.73,220.69 L960.48,220.69 '/> <g transform="translate(674.17,224.27)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 10000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,170.63 L968.73,170.63 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,170.63 L690.11,170.63 M968.73,170.63 L960.48,170.63 '/> <g transform="translate(674.17,174.21)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 12000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,120.57 L968.73,120.57 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,120.57 L690.11,120.57 M968.73,120.57 L960.48,120.57 '/> <g transform="translate(674.17,124.15)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 14000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M681.86,70.51 L968.73,70.51 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,70.51 L690.11,70.51 M968.73,70.51 L960.48,70.51 '/> <g transform="translate(674.17,74.09)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 16000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M825.30,420.94 L825.30,412.69 M825.30,70.51 L825.30,78.76 '/> <g transform="translate(823.51,431.73) rotate(30.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="start">
<text><tspan font-family="sans" >v0.5.0</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,70.51 L681.86,420.94 L968.73,420.94 L968.73,70.51 L681.86,70.51 Z '/></g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g id="gnuplot_plot_1b" ><title>parrhesia-pg</title>
<g fill="none" color="white" stroke="black" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(1137.91,82.34)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-pg</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 37, 99, 235)' d='M1145.60,78.76 L1184.61,78.76 M825.30,310.40 '/> <use xlink:href='#gpPt6' transform='translate(825.30,310.40) scale(4.12)' color='rgb( 37, 99, 235)'/>
<use xlink:href='#gpPt6' transform='translate(1165.10,78.76) scale(4.12)' color='rgb( 37, 99, 235)'/>
</g>
</g>
<g id="gnuplot_plot_2b" ><title>parrhesia-memory</title>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(1137.91,98.84)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-memory</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 22, 163, 74)' d='M1145.60,95.26 L1184.61,95.26 M825.30,106.39 '/> <use xlink:href='#gpPt8' transform='translate(825.30,106.39) scale(4.12)' color='rgb( 22, 163, 74)'/>
<use xlink:href='#gpPt8' transform='translate(1165.10,95.26) scale(4.12)' color='rgb( 22, 163, 74)'/>
</g>
</g>
<g id="gnuplot_plot_3b" ><title>nostr-rs-relay (avg)</title>
<g fill="none" color="white" stroke="rgb( 22, 163, 74)" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(1137.91,115.34)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >nostr-rs-relay (avg)</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb(234, 88, 12)' stroke-dasharray='3.8,6.0' d='M1145.60,111.76 L1184.61,111.76 M825.30,395.02 '/> <use xlink:href='#gpPt4' transform='translate(825.30,395.02) scale(3.30)' color='rgb(234, 88, 12)'/>
<use xlink:href='#gpPt4' transform='translate(1165.10,111.76) scale(3.30)' color='rgb(234, 88, 12)'/>
</g>
</g>
<g fill="none" color="white" stroke="rgb(234, 88, 12)" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M681.86,70.51 L681.86,420.94 L968.73,420.94 L968.73,70.51 L681.86,70.51 Z '/> <g transform="translate(617.58,245.73) rotate(270.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >TPS</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(825.29,49.34)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >Req Throughput (TPS) — higher is better</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,860.44 L368.73,860.44 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,860.44 L97.80,860.44 M368.73,860.44 L360.48,860.44 '/> <g transform="translate(81.86,864.02)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 60000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,821.50 L368.73,821.50 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,821.50 L97.80,821.50 M368.73,821.50 L360.48,821.50 '/> <g transform="translate(81.86,825.08)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 70000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,782.56 L368.73,782.56 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,782.56 L97.80,782.56 M368.73,782.56 L360.48,782.56 '/> <g transform="translate(81.86,786.14)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 80000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,743.63 L368.73,743.63 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,743.63 L97.80,743.63 M368.73,743.63 L360.48,743.63 '/> <g transform="translate(81.86,747.21)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 90000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,704.69 L368.73,704.69 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,704.69 L97.80,704.69 M368.73,704.69 L360.48,704.69 '/> <g transform="translate(81.86,708.27)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 100000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,665.75 L368.73,665.75 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,665.75 L97.80,665.75 M368.73,665.75 L360.48,665.75 '/> <g transform="translate(81.86,669.33)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 110000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,626.81 L368.73,626.81 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,626.81 L97.80,626.81 M368.73,626.81 L360.48,626.81 '/> <g transform="translate(81.86,630.39)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 120000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,587.88 L368.73,587.88 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,587.88 L97.80,587.88 M368.73,587.88 L360.48,587.88 '/> <g transform="translate(81.86,591.46)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 130000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,548.94 L368.73,548.94 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,548.94 L97.80,548.94 M368.73,548.94 L360.48,548.94 '/> <g transform="translate(81.86,552.52)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 140000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M89.55,510.00 L368.73,510.00 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,510.00 L97.80,510.00 M368.73,510.00 L360.48,510.00 '/> <g transform="translate(81.86,513.58)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 150000</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M229.14,860.44 L229.14,852.19 M229.14,510.00 L229.14,518.25 '/> <g transform="translate(227.35,871.23) rotate(30.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="start">
<text><tspan font-family="sans" >v0.5.0</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,510.00 L89.55,860.44 L368.73,860.44 L368.73,510.00 L89.55,510.00 Z '/></g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g id="gnuplot_plot_1c" ><title>parrhesia-pg</title>
<g fill="none" color="white" stroke="black" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(537.91,521.83)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-pg</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 37, 99, 235)' d='M545.60,518.25 L584.61,518.25 M229.14,844.75 '/> <use xlink:href='#gpPt6' transform='translate(229.14,844.75) scale(4.12)' color='rgb( 37, 99, 235)'/>
<use xlink:href='#gpPt6' transform='translate(565.10,518.25) scale(4.12)' color='rgb( 37, 99, 235)'/>
</g>
</g>
<g id="gnuplot_plot_2c" ><title>parrhesia-memory</title>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(537.91,538.33)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-memory</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 22, 163, 74)' d='M545.60,534.75 L584.61,534.75 M229.14,729.39 '/> <use xlink:href='#gpPt8' transform='translate(229.14,729.39) scale(4.12)' color='rgb( 22, 163, 74)'/>
<use xlink:href='#gpPt8' transform='translate(565.10,534.75) scale(4.12)' color='rgb( 22, 163, 74)'/>
</g>
</g>
<g id="gnuplot_plot_3c" ><title>nostr-rs-relay (avg)</title>
<g fill="none" color="white" stroke="rgb( 22, 163, 74)" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(537.91,554.83)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >nostr-rs-relay (avg)</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb(234, 88, 12)' stroke-dasharray='3.8,6.0' d='M545.60,551.25 L584.61,551.25 M229.14,545.95 '/> <use xlink:href='#gpPt4' transform='translate(229.14,545.95) scale(3.30)' color='rgb(234, 88, 12)'/>
<use xlink:href='#gpPt4' transform='translate(565.10,551.25) scale(3.30)' color='rgb(234, 88, 12)'/>
</g>
</g>
<g fill="none" color="white" stroke="rgb(234, 88, 12)" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M89.55,510.00 L89.55,860.44 L368.73,860.44 L368.73,510.00 L89.55,510.00 Z '/> <g transform="translate(17.58,685.22) rotate(270.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >TPS</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(229.14,488.83)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >Echo Throughput (TPS) — higher is better</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M666.48,860.44 L968.73,860.44 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,860.44 L674.73,860.44 M968.73,860.44 L960.48,860.44 '/> <g transform="translate(658.79,864.02)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 7</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M666.48,790.35 L968.73,790.35 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,790.35 L674.73,790.35 M968.73,790.35 L960.48,790.35 '/> <g transform="translate(658.79,793.93)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 7.5</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M666.48,720.26 L968.73,720.26 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,720.26 L674.73,720.26 M968.73,720.26 L960.48,720.26 '/> <g transform="translate(658.79,723.84)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 8</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M666.48,650.18 L968.73,650.18 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,650.18 L674.73,650.18 M968.73,650.18 L960.48,650.18 '/> <g transform="translate(658.79,653.76)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 8.5</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M666.48,580.09 L968.73,580.09 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,580.09 L674.73,580.09 M968.73,580.09 L960.48,580.09 '/> <g transform="translate(658.79,583.67)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 9</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="gray" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='gray' stroke-dasharray='2,4' class="gridline" d='M666.48,510.00 L968.73,510.00 '/></g>
<g fill="none" color="gray" stroke="gray" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,510.00 L674.73,510.00 M968.73,510.00 L960.48,510.00 '/> <g transform="translate(658.79,513.58)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" > 9.5</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M817.61,860.44 L817.61,852.19 M817.61,510.00 L817.61,518.25 '/> <g transform="translate(815.82,871.23) rotate(30.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="start">
<text><tspan font-family="sans" >v0.5.0</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,510.00 L666.48,860.44 L968.73,860.44 L968.73,510.00 L666.48,510.00 Z '/></g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g id="gnuplot_plot_1d" ><title>parrhesia-pg</title>
<g fill="none" color="white" stroke="black" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(1137.91,521.83)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-pg</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 37, 99, 235)' d='M1145.60,518.25 L1184.61,518.25 M817.61,533.36 '/> <use xlink:href='#gpPt6' transform='translate(817.61,533.36) scale(4.12)' color='rgb( 37, 99, 235)'/>
<use xlink:href='#gpPt6' transform='translate(1165.10,518.25) scale(4.12)' color='rgb( 37, 99, 235)'/>
</g>
</g>
<g id="gnuplot_plot_2d" ><title>parrhesia-memory</title>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(1137.91,538.33)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >parrhesia-memory</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb( 22, 163, 74)' d='M1145.60,534.75 L1184.61,534.75 M817.61,766.99 '/> <use xlink:href='#gpPt8' transform='translate(817.61,766.99) scale(4.12)' color='rgb( 22, 163, 74)'/>
<use xlink:href='#gpPt8' transform='translate(1165.10,534.75) scale(4.12)' color='rgb( 22, 163, 74)'/>
</g>
</g>
<g id="gnuplot_plot_3d" ><title>nostr-rs-relay (avg)</title>
<g fill="none" color="white" stroke="rgb( 22, 163, 74)" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(1137.91,554.83)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="end">
<text><tspan font-family="sans" >nostr-rs-relay (avg)</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.50" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='rgb(234, 88, 12)' stroke-dasharray='3.8,6.0' d='M1145.60,551.25 L1184.61,551.25 M817.61,860.44 '/> <use xlink:href='#gpPt4' transform='translate(817.61,860.44) scale(3.30)' color='rgb(234, 88, 12)'/>
<use xlink:href='#gpPt4' transform='translate(1165.10,551.25) scale(3.30)' color='rgb(234, 88, 12)'/>
</g>
</g>
<g fill="none" color="white" stroke="rgb(234, 88, 12)" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<path stroke='black' d='M666.48,510.00 L666.48,860.44 L968.73,860.44 L968.73,510.00 L666.48,510.00 Z '/> <g transform="translate(617.58,685.22) rotate(270.00)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >ms</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
<g transform="translate(817.60,488.83)" stroke="none" fill="black" font-family="sans" font-size="11.00" text-anchor="middle">
<text><tspan font-family="sans" >Connect Avg Latency (ms) — lower is better</tspan></text>
</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 53 KiB

1
bench/history.jsonl Normal file
View File

@@ -0,0 +1 @@
{"timestamp":"2026-03-18T20:13:21Z","machine_id":"squirrel","git_tag":"v0.5.0","git_commit":"970cee2","runs":3,"servers":{"parrhesia-pg":{"connect_avg_ms":9.333333333333334,"connect_max_ms":12.333333333333334,"echo_tps":64030.333333333336,"echo_mibs":35.06666666666666,"event_tps":5015.333333333333,"event_mibs":3.4,"req_tps":6416.333333333333,"req_mibs":42.43333333333334},"parrhesia-memory":{"connect_avg_ms":7.666666666666667,"connect_max_ms":9.666666666666666,"echo_tps":93656.33333333333,"echo_mibs":51.26666666666667,"event_tps":1505.3333333333333,"event_mibs":1,"req_tps":14566.666666666666,"req_mibs":94.23333333333335},"nostr-rs-relay":{"connect_avg_ms":7,"connect_max_ms":10.333333333333334,"echo_tps":140767,"echo_mibs":77.06666666666666,"event_tps":2293.6666666666665,"event_mibs":1.5,"req_tps":3035.6666666666665,"req_mibs":19.23333333333333}}}

View File

@@ -101,6 +101,8 @@ in {
nostr-bench
# Nostr reference servers
nostr-rs-relay
# Benchmark graph
gnuplot
]
++ lib.optionals pkgs.stdenv.hostPlatform.isx86_64 [
strfry

View File

@@ -26,7 +26,7 @@ defmodule Parrhesia.MixProject do
defp elixirc_paths(_env), do: ["lib"]
def cli do
[preferred_envs: [precommit: :test, bench: :test]]
[preferred_envs: [precommit: :test, bench: :test, "bench.update": :test]]
end
# Run "mix help deps" to learn about dependencies.
@@ -71,6 +71,7 @@ defmodule Parrhesia.MixProject do
"test.node_sync_e2e": ["cmd ./scripts/run_node_sync_e2e.sh"],
"test.node_sync_docker_e2e": ["cmd ./scripts/run_node_sync_docker_e2e.sh"],
bench: ["cmd ./scripts/run_bench_compare.sh"],
"bench.update": ["cmd ./scripts/run_bench_update.sh"],
# cov: ["cmd mix coveralls.lcov"],
lint: ["format --check-formatted", "credo"],
precommit: [

View File

@@ -477,4 +477,35 @@ for (let i = 0; i < runs; i += 1) {
}
console.log(line);
}
// Structured JSON output for automation (bench:update pipeline)
if (process.env.BENCH_JSON_OUT) {
const jsonSummary = {};
const serverKeys = [
["parrhesia-pg", "parrhesia"],
["parrhesia-memory", "parrhesiaMemory"],
];
if (hasStrfry) serverKeys.push(["strfry", "strfry"]);
if (hasNostrRs) serverKeys.push(["nostr-rs-relay", "nostrRsRelay"]);
for (const [outputKey, summaryKey] of serverKeys) {
const s = summary[summaryKey];
jsonSummary[outputKey] = {
connect_avg_ms: s.connectAvgMs,
connect_max_ms: s.connectMaxMs,
echo_tps: s.echoTps,
echo_mibs: s.echoSizeMiBS,
event_tps: s.eventTps,
event_mibs: s.eventSizeMiBS,
req_tps: s.reqTps,
req_mibs: s.reqSizeMiBS,
};
}
fs.writeFileSync(
process.env.BENCH_JSON_OUT,
JSON.stringify(jsonSummary, null, 2) + "\n",
"utf8"
);
}
NODE

329
scripts/run_bench_update.sh Executable file
View File

@@ -0,0 +1,329 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
usage() {
cat <<'EOF'
usage:
./scripts/run_bench_update.sh
Runs the benchmark suite (3 runs by default), then:
1) Appends structured results to bench/history.jsonl
2) Generates bench/chart.svg via gnuplot
3) Updates the comparison table in README.md
Environment:
PARRHESIA_BENCH_RUNS Number of runs (default: 3)
PARRHESIA_BENCH_MACHINE_ID Machine identifier (default: hostname -s)
All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded.
EOF
}
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
usage
exit 0
fi
# --- Configuration -----------------------------------------------------------
BENCH_DIR="$ROOT_DIR/bench"
HISTORY_FILE="$BENCH_DIR/history.jsonl"
CHART_FILE="$BENCH_DIR/chart.svg"
GNUPLOT_TEMPLATE="$BENCH_DIR/chart.gnuplot"
MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}"
GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')"
GIT_COMMIT="$(git rev-parse --short=7 HEAD)"
TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
RUNS="${PARRHESIA_BENCH_RUNS:-3}"
mkdir -p "$BENCH_DIR"
WORK_DIR="$(mktemp -d)"
trap 'rm -rf "$WORK_DIR"' EXIT
JSON_OUT="$WORK_DIR/bench_summary.json"
RAW_OUTPUT="$WORK_DIR/bench_output.txt"
# --- Phase 1: Run benchmarks -------------------------------------------------
echo "Running ${RUNS}-run benchmark suite..."
PARRHESIA_BENCH_RUNS="$RUNS" \
BENCH_JSON_OUT="$JSON_OUT" \
./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT"
if [[ ! -f "$JSON_OUT" ]]; then
echo "Benchmark JSON output not found at $JSON_OUT" >&2
exit 1
fi
# --- Phase 2: Append to history ----------------------------------------------
echo "Appending to history..."
node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE'
const fs = require("node:fs");
const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv;
const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
const entry = {
timestamp,
machine_id: machineId,
git_tag: gitTag,
git_commit: gitCommit,
runs: Number(runsStr),
servers,
};
fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8");
console.log(" entry: " + gitTag + " (" + gitCommit + ") on " + machineId);
NODE
# --- Phase 3: Generate chart --------------------------------------------------
echo "Generating chart..."
node - "$HISTORY_FILE" "$MACHINE_ID" "$WORK_DIR" <<'NODE'
const fs = require("node:fs");
const path = require("node:path");
const [, , historyFile, machineId, workDir] = process.argv;
if (!fs.existsSync(historyFile)) {
console.log(" no history file, skipping chart generation");
process.exit(0);
}
const lines = fs.readFileSync(historyFile, "utf8")
.split("\n")
.filter(l => l.trim().length > 0)
.map(l => JSON.parse(l));
// Filter to current machine
const entries = lines.filter(e => e.machine_id === machineId);
if (entries.length === 0) {
console.log(" no history entries for machine '" + machineId + "', skipping chart");
process.exit(0);
}
// Sort chronologically, deduplicate by tag (latest wins)
entries.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
const byTag = new Map();
for (const e of entries) {
byTag.set(e.git_tag, e);
}
const deduped = [...byTag.values()];
// Determine which non-parrhesia servers are present
const baselineServerNames = ["strfry", "nostr-rs-relay"];
const presentBaselines = baselineServerNames.filter(srv =>
deduped.some(e => e.servers[srv])
);
// Compute averages for baseline servers (constant horizontal lines)
const baselineAvg = {};
for (const srv of presentBaselines) {
const vals = deduped.filter(e => e.servers[srv]).map(e => e.servers[srv]);
baselineAvg[srv] = {};
for (const metric of Object.keys(vals[0])) {
const valid = vals.map(v => v[metric]).filter(Number.isFinite);
baselineAvg[srv][metric] = valid.length > 0
? valid.reduce((a, b) => a + b, 0) / valid.length
: NaN;
}
}
// Metrics to chart
const chartMetrics = [
{ key: "event_tps", label: "Event Throughput (TPS) — higher is better", file: "event_tps.tsv", ylabel: "TPS" },
{ key: "req_tps", label: "Req Throughput (TPS) — higher is better", file: "req_tps.tsv", ylabel: "TPS" },
{ key: "echo_tps", label: "Echo Throughput (TPS) — higher is better", file: "echo_tps.tsv", ylabel: "TPS" },
{ key: "connect_avg_ms", label: "Connect Avg Latency (ms) — lower is better", file: "connect_avg_ms.tsv", ylabel: "ms" },
];
// Write per-metric TSV files
for (const cm of chartMetrics) {
const header = ["tag", "parrhesia-pg", "parrhesia-memory"];
for (const srv of presentBaselines) header.push(srv);
const rows = [header.join("\t")];
for (const e of deduped) {
const row = [
e.git_tag,
e.servers["parrhesia-pg"]?.[cm.key] ?? "NaN",
e.servers["parrhesia-memory"]?.[cm.key] ?? "NaN",
];
for (const srv of presentBaselines) {
row.push(baselineAvg[srv]?.[cm.key] ?? "NaN");
}
rows.push(row.join("\t"));
}
fs.writeFileSync(path.join(workDir, cm.file), rows.join("\n") + "\n", "utf8");
}
// Generate gnuplot plot commands (handles variable column counts)
const serverLabels = ["parrhesia-pg", "parrhesia-memory"];
for (const srv of presentBaselines) serverLabels.push(srv + " (avg)");
const plotLines = [];
for (const cm of chartMetrics) {
const dataFile = `data_dir."/${cm.file}"`;
plotLines.push(`set title "${cm.label}"`);
plotLines.push(`set ylabel "${cm.ylabel}"`);
const plotParts = [];
// Column 2 = parrhesia-pg, 3 = parrhesia-memory, 4+ = baselines
plotParts.push(`${dataFile} using 0:2:xtic(1) lt 1 title "${serverLabels[0]}"`);
plotParts.push(`'' using 0:3 lt 2 title "${serverLabels[1]}"`);
for (let i = 0; i < presentBaselines.length; i++) {
plotParts.push(`'' using 0:${4 + i} lt ${3 + i} title "${serverLabels[2 + i]}"`);
}
plotLines.push("plot " + plotParts.join(", \\\n "));
plotLines.push("");
}
fs.writeFileSync(
path.join(workDir, "plot_commands.gnuplot"),
plotLines.join("\n") + "\n",
"utf8"
);
console.log(" " + deduped.length + " tag(s), " + presentBaselines.length + " baseline server(s)");
NODE
if [[ -f "$WORK_DIR/plot_commands.gnuplot" ]]; then
gnuplot \
-e "data_dir='$WORK_DIR'" \
-e "output_file='$CHART_FILE'" \
"$GNUPLOT_TEMPLATE"
echo " chart written to $CHART_FILE"
else
echo " chart generation skipped (no data for this machine)"
fi
# --- Phase 4: Update README.md -----------------------------------------------
echo "Updating README.md..."
node - "$JSON_OUT" "$ROOT_DIR/README.md" <<'NODE'
const fs = require("node:fs");
const [, , jsonOut, readmePath] = process.argv;
const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
const readme = fs.readFileSync(readmePath, "utf8");
const pg = servers["parrhesia-pg"];
const mem = servers["parrhesia-memory"];
const strfry = servers["strfry"];
const nostrRs = servers["nostr-rs-relay"];
function toFixed(v, d = 2) {
return Number.isFinite(v) ? v.toFixed(d) : "n/a";
}
function ratio(base, other) {
if (!Number.isFinite(base) || !Number.isFinite(other) || base === 0) return "n/a";
return (other / base).toFixed(2) + "x";
}
function boldIf(ratioStr, lowerIsBetter) {
if (ratioStr === "n/a") return ratioStr;
const num = parseFloat(ratioStr);
const better = lowerIsBetter ? num < 1 : num > 1;
return better ? "**" + ratioStr + "**" : ratioStr;
}
const metricRows = [
["connect avg latency (ms) \u2193", "connect_avg_ms", true],
["connect max latency (ms) \u2193", "connect_max_ms", true],
["echo throughput (TPS) \u2191", "echo_tps", false],
["echo throughput (MiB/s) \u2191", "echo_mibs", false],
["event throughput (TPS) \u2191", "event_tps", false],
["event throughput (MiB/s) \u2191", "event_mibs", false],
["req throughput (TPS) \u2191", "req_tps", false],
["req throughput (MiB/s) \u2191", "req_mibs", false],
];
const hasStrfry = !!strfry;
const hasNostrRs = !!nostrRs;
// Build header
const header = ["metric", "parrhesia-pg", "parrhesia-mem"];
if (hasStrfry) header.push("strfry");
if (hasNostrRs) header.push("nostr-rs-relay");
header.push("mem/pg");
if (hasStrfry) header.push("strfry/pg");
if (hasNostrRs) header.push("nostr-rs/pg");
const alignRow = ["---"];
for (let i = 1; i < header.length; i++) alignRow.push("---:");
const rows = metricRows.map(([label, key, lowerIsBetter]) => {
const row = [label, toFixed(pg[key]), toFixed(mem[key])];
if (hasStrfry) row.push(toFixed(strfry[key]));
if (hasNostrRs) row.push(toFixed(nostrRs[key]));
row.push(boldIf(ratio(pg[key], mem[key]), lowerIsBetter));
if (hasStrfry) row.push(boldIf(ratio(pg[key], strfry[key]), lowerIsBetter));
if (hasNostrRs) row.push(boldIf(ratio(pg[key], nostrRs[key]), lowerIsBetter));
return row;
});
const tableLines = [
"| " + header.join(" | ") + " |",
"| " + alignRow.join(" | ") + " |",
...rows.map(r => "| " + r.join(" | ") + " |"),
];
// Replace the first markdown table in the ## Benchmark section
const readmeLines = readme.split("\n");
const benchIdx = readmeLines.findIndex(l => /^## Benchmark/.test(l));
if (benchIdx === -1) {
console.error("Could not find '## Benchmark' section in README.md");
process.exit(1);
}
let tableStart = -1;
let tableEnd = -1;
for (let i = benchIdx + 1; i < readmeLines.length; i++) {
if (readmeLines[i].startsWith("|")) {
if (tableStart === -1) tableStart = i;
tableEnd = i;
} else if (tableStart !== -1) {
break;
}
}
if (tableStart === -1) {
console.error("Could not find markdown table in ## Benchmark section");
process.exit(1);
}
const before = readmeLines.slice(0, tableStart);
const after = readmeLines.slice(tableEnd + 1);
const updated = [...before, ...tableLines, ...after].join("\n");
fs.writeFileSync(readmePath, updated, "utf8");
console.log(" table updated (" + tableLines.length + " rows)");
NODE
# --- Done ---------------------------------------------------------------------
echo
echo "Benchmark update complete. Files changed:"
echo " $HISTORY_FILE"
echo " $CHART_FILE"
echo " $ROOT_DIR/README.md"
echo
echo "Review with: git diff"