diff --git a/bench/chart.svg b/bench/chart.svg
index f993cdf..81df9e4 100644
--- a/bench/chart.svg
+++ b/bench/chart.svg
@@ -150,8 +150,28 @@
4000
+
+
+ v0.2.0
+
+
+
+
+ v0.3.0
+
+
+ v0.4.0
+
+
+
+
+ v0.5.0
+
+
+
+
v0.6.0
@@ -170,7 +190,11 @@
-
+
+
+
+
+
@@ -181,7 +205,11 @@
-
+
+
+
+
+
@@ -194,7 +222,11 @@
-
+
+
+
+
+
@@ -205,7 +237,11 @@
-
+
+
+
+
+
@@ -333,8 +369,28 @@
7000
+
+
+ v0.2.0
+
+
+
+
+ v0.3.0
+
+
+ v0.4.0
+
+
+
+
+ v0.5.0
+
+
+
+
v0.6.0
@@ -353,7 +409,11 @@
-
+
+
+
+
+
@@ -364,7 +424,11 @@
-
+
+
+
+
+
@@ -377,7 +441,11 @@
-
+
+
+
+
+
@@ -388,7 +456,11 @@
-
+
+
+
+
+
@@ -516,8 +588,28 @@
180000
+
+
+ v0.2.0
+
+
+
+
+ v0.3.0
+
+
+ v0.4.0
+
+
+
+
+ v0.5.0
+
+
+
+
v0.6.0
@@ -536,7 +628,11 @@
-
+
+
+
+
+
@@ -547,7 +643,11 @@
-
+
+
+
+
+
@@ -560,7 +660,11 @@
-
+
+
+
+
+
@@ -571,7 +675,11 @@
-
+
+
+
+
+
@@ -613,11 +721,11 @@
-
+
-
+
5
@@ -625,13 +733,26 @@
+
+
+
+
+
+
+ 10
+
+
+
+
+
+
- 10
+ 15
@@ -639,12 +760,25 @@
-
+
-
- 15
+
+ 20
+
+
+
+
+
+
+
+
+
+
+
+
+ 25
@@ -657,7 +791,7 @@
- 20
+ 30
@@ -665,12 +799,25 @@
-
+
-
- 25
+
+ 35
+
+
+
+
+
+
+
+
+
+
+
+
+ 40
@@ -683,11 +830,31 @@
- 30
+ 45
+
+
+
+
+ v0.2.0
+
+
+
+
+ v0.3.0
+ v0.4.0
+
+
+
+
+ v0.5.0
+
+
+
+
v0.6.0
@@ -706,7 +873,11 @@
-
+
+
+
+
+
@@ -717,7 +888,11 @@
-
+
+
+
+
+
@@ -730,7 +905,11 @@
-
+
+
+
+
+
@@ -741,7 +920,11 @@
-
+
+
+
+
+
diff --git a/bench/history.jsonl b/bench/history.jsonl
index 7182523..77c9cdf 100644
--- a/bench/history.jsonl
+++ b/bench/history.jsonl
@@ -1,5 +1,5 @@
-{"timestamp":"2026-03-18T21:35:03Z","machine_id":"agent","git_tag":"v0.6.0","git_commit":"7b337d9","runs":3,"versions":{"parrhesia":"0.6.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":26.666666666666668,"connect_max_ms":45.333333333333336,"echo_tps":68100.33333333333,"echo_mibs":37.233333333333334,"event_tps":1647.3333333333333,"event_mibs":1.0666666666666667,"req_tps":3576.6666666666665,"req_mibs":18.833333333333332},"parrhesia-memory":{"connect_avg_ms":14.666666666666666,"connect_max_ms":24.333333333333332,"echo_tps":55978,"echo_mibs":30.633333333333336,"event_tps":882,"event_mibs":0.5666666666666668,"req_tps":6888,"req_mibs":36.06666666666666},"strfry":{"connect_avg_ms":3,"connect_max_ms":4.666666666666667,"echo_tps":67718.33333333333,"echo_mibs":37.86666666666667,"event_tps":3548.3333333333335,"event_mibs":2.3,"req_tps":1808,"req_mibs":11.699999999999998},"nostr-rs-relay":{"connect_avg_ms":2,"connect_max_ms":3.3333333333333335,"echo_tps":166178,"echo_mibs":91.03333333333335,"event_tps":787,"event_mibs":0.5,"req_tps":860.6666666666666,"req_mibs":2.4}}}
{"timestamp":"2026-03-18T22:14:37Z","machine_id":"agent","git_tag":"v0.2.0","git_commit":"b20dbf6","runs":3,"versions":{"parrhesia":"0.2.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":14.666666666666666,"connect_max_ms":25.666666666666668,"echo_tps":77133,"echo_mibs":42.233333333333334,"event_tps":1602.6666666666667,"event_mibs":1.0666666666666667,"req_tps":2418,"req_mibs":12.5},"parrhesia-memory":{"connect_avg_ms":9,"connect_max_ms":16,"echo_tps":64218.333333333336,"echo_mibs":35.166666666666664,"event_tps":1578.3333333333333,"event_mibs":1,"req_tps":2431.3333333333335,"req_mibs":12.633333333333333},"strfry":{"connect_avg_ms":3.3333333333333335,"connect_max_ms":6,"echo_tps":63682.666666666664,"echo_mibs":35.6,"event_tps":3477.3333333333335,"event_mibs":2.2333333333333334,"req_tps":1804,"req_mibs":11.733333333333334},"nostr-rs-relay":{"connect_avg_ms":2.6666666666666665,"connect_max_ms":4.333333333333333,"echo_tps":160009,"echo_mibs":87.63333333333333,"event_tps":762,"event_mibs":0.4666666666666666,"req_tps":831,"req_mibs":2.2333333333333334}}}
{"timestamp":"2026-03-18T22:22:12Z","machine_id":"agent","git_tag":"v0.3.0","git_commit":"8c8d5a8","runs":3,"versions":{"parrhesia":"0.3.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":13,"connect_max_ms":21.666666666666668,"echo_tps":70703.33333333333,"echo_mibs":38.7,"event_tps":1970.6666666666667,"event_mibs":1.3,"req_tps":3614,"req_mibs":20.966666666666665},"parrhesia-memory":{"connect_avg_ms":13,"connect_max_ms":22.333333333333332,"echo_tps":60452.333333333336,"echo_mibs":33.1,"event_tps":1952.6666666666667,"event_mibs":1.3,"req_tps":3616,"req_mibs":20.766666666666666},"strfry":{"connect_avg_ms":3.6666666666666665,"connect_max_ms":6,"echo_tps":63128.666666666664,"echo_mibs":35.300000000000004,"event_tps":3442,"event_mibs":2.2333333333333334,"req_tps":1804,"req_mibs":11.699999999999998},"nostr-rs-relay":{"connect_avg_ms":2,"connect_max_ms":3.3333333333333335,"echo_tps":164995.33333333334,"echo_mibs":90.36666666666667,"event_tps":761.6666666666666,"event_mibs":0.5,"req_tps":846.3333333333334,"req_mibs":2.333333333333333}}}
{"timestamp":"2026-03-18T22:30:08Z","machine_id":"agent","git_tag":"v0.4.0","git_commit":"b86b5db","runs":3,"versions":{"parrhesia":"0.4.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":11.333333333333334,"connect_max_ms":20.666666666666668,"echo_tps":69139.33333333333,"echo_mibs":37.833333333333336,"event_tps":1938.6666666666667,"event_mibs":1.3,"req_tps":4619.666666666667,"req_mibs":26.266666666666666},"parrhesia-memory":{"connect_avg_ms":10,"connect_max_ms":17.333333333333332,"echo_tps":62715.333333333336,"echo_mibs":34.333333333333336,"event_tps":1573,"event_mibs":1.0333333333333334,"req_tps":4768,"req_mibs":23.733333333333334},"strfry":{"connect_avg_ms":3.3333333333333335,"connect_max_ms":6,"echo_tps":60956.666666666664,"echo_mibs":34.06666666666667,"event_tps":3380.6666666666665,"event_mibs":2.2,"req_tps":1820.3333333333333,"req_mibs":11.800000000000002},"nostr-rs-relay":{"connect_avg_ms":2.6666666666666665,"connect_max_ms":4.333333333333333,"echo_tps":161165.33333333334,"echo_mibs":88.26666666666665,"event_tps":768,"event_mibs":0.5,"req_tps":847.3333333333334,"req_mibs":2.3000000000000003}}}
{"timestamp":"2026-03-18T22:36:37Z","machine_id":"agent","git_tag":"v0.5.0","git_commit":"e557eba","runs":3,"versions":{"parrhesia":"0.5.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":34.666666666666664,"connect_max_ms":61.666666666666664,"echo_tps":72441,"echo_mibs":39.666666666666664,"event_tps":1897.3333333333333,"event_mibs":1.2333333333333334,"req_tps":13.333333333333334,"req_mibs":0.03333333333333333},"parrhesia-memory":{"connect_avg_ms":43.333333333333336,"connect_max_ms":74.66666666666667,"echo_tps":62704.666666666664,"echo_mibs":34.300000000000004,"event_tps":1370,"event_mibs":0.8666666666666667,"req_tps":47,"req_mibs":0.16666666666666666},"strfry":{"connect_avg_ms":2.6666666666666665,"connect_max_ms":4.666666666666667,"echo_tps":61189.333333333336,"echo_mibs":34.2,"event_tps":3426.6666666666665,"event_mibs":2.2,"req_tps":1811.3333333333333,"req_mibs":11.766666666666666},"nostr-rs-relay":{"connect_avg_ms":2.6666666666666665,"connect_max_ms":4,"echo_tps":152654.33333333334,"echo_mibs":83.63333333333333,"event_tps":772.6666666666666,"event_mibs":0.5,"req_tps":878.3333333333334,"req_mibs":2.4}}}
+{"timestamp":"2026-03-18T21:35:03Z","machine_id":"agent","git_tag":"v0.6.0","git_commit":"7b337d9","runs":3,"versions":{"parrhesia":"0.6.0","strfry":"strfry 1.0.4 (nixpkgs)","nostr-rs-relay":"nostr-rs-relay 0.9.0","nostr-bench":"nostr-bench 0.4.0"},"servers":{"parrhesia-pg":{"connect_avg_ms":26.666666666666668,"connect_max_ms":45.333333333333336,"echo_tps":68100.33333333333,"echo_mibs":37.233333333333334,"event_tps":1647.3333333333333,"event_mibs":1.0666666666666667,"req_tps":3576.6666666666665,"req_mibs":18.833333333333332},"parrhesia-memory":{"connect_avg_ms":14.666666666666666,"connect_max_ms":24.333333333333332,"echo_tps":55978,"echo_mibs":30.633333333333336,"event_tps":882,"event_mibs":0.5666666666666668,"req_tps":6888,"req_mibs":36.06666666666666},"strfry":{"connect_avg_ms":3,"connect_max_ms":4.666666666666667,"echo_tps":67718.33333333333,"echo_mibs":37.86666666666667,"event_tps":3548.3333333333335,"event_mibs":2.3,"req_tps":1808,"req_mibs":11.699999999999998},"nostr-rs-relay":{"connect_avg_ms":2,"connect_max_ms":3.3333333333333335,"echo_tps":166178,"echo_mibs":91.03333333333335,"event_tps":787,"event_mibs":0.5,"req_tps":860.6666666666666,"req_mibs":2.4}}}
diff --git a/mix.exs b/mix.exs
index 458816d..c1d655a 100644
--- a/mix.exs
+++ b/mix.exs
@@ -26,7 +26,15 @@ defmodule Parrhesia.MixProject do
defp elixirc_paths(_env), do: ["lib"]
def cli do
- [preferred_envs: [precommit: :test, bench: :test, "bench.update": :test]]
+ [
+ preferred_envs: [
+ precommit: :test,
+ bench: :test,
+ "bench.collect": :test,
+ "bench.update": :test,
+ "bench.at": :test
+ ]
+ ]
end
# Run "mix help deps" to learn about dependencies.
@@ -71,7 +79,9 @@ defmodule Parrhesia.MixProject do
"test.node_sync_e2e": ["cmd ./scripts/run_node_sync_e2e.sh"],
"test.node_sync_docker_e2e": ["cmd ./scripts/run_node_sync_docker_e2e.sh"],
bench: ["cmd ./scripts/run_bench_compare.sh"],
+ "bench.collect": ["cmd ./scripts/run_bench_collect.sh"],
"bench.update": ["cmd ./scripts/run_bench_update.sh"],
+ "bench.at": ["cmd ./scripts/run_bench_at_ref.sh"],
# cov: ["cmd mix coveralls.lcov"],
lint: ["format --check-formatted", "credo"],
precommit: [
diff --git a/scripts/run_bench_at_ref.sh b/scripts/run_bench_at_ref.sh
index 0db8d20..12e7e81 100755
--- a/scripts/run_bench_at_ref.sh
+++ b/scripts/run_bench_at_ref.sh
@@ -67,7 +67,7 @@ cd "$WORKTREE_DIR"
# Always copy latest benchmark scripts to ensure consistency
echo "Copying latest benchmark infrastructure from current..."
mkdir -p scripts bench
-cp "$ROOT_DIR/scripts/run_bench_update.sh" scripts/
+cp "$ROOT_DIR/scripts/run_bench_collect.sh" scripts/
cp "$ROOT_DIR/scripts/run_bench_compare.sh" scripts/
cp "$ROOT_DIR/scripts/run_nostr_bench.sh" scripts/
if [[ -f "$ROOT_DIR/scripts/run_nostr_bench_strfry.sh" ]]; then
@@ -76,9 +76,6 @@ fi
if [[ -f "$ROOT_DIR/scripts/run_nostr_bench_nostr_rs_relay.sh" ]]; then
cp "$ROOT_DIR/scripts/run_nostr_bench_nostr_rs_relay.sh" scripts/
fi
-if [[ -f "$ROOT_DIR/bench/chart.gnuplot" ]]; then
- cp "$ROOT_DIR/bench/chart.gnuplot" bench/
-fi
echo
echo "Installing dependencies..."
@@ -90,11 +87,10 @@ echo
RUNS="${PARRHESIA_BENCH_RUNS:-3}"
-# Run the benchmark update script which will append to history.jsonl
-# Allow it to fail (e.g., README update might fail on old versions) but continue
+# Run the benchmark collect script which will append to history.jsonl
PARRHESIA_BENCH_RUNS="$RUNS" \
PARRHESIA_BENCH_MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-}" \
- ./scripts/run_bench_update.sh || echo "Benchmark script exited with error (may be expected for old versions)"
+ ./scripts/run_bench_collect.sh
# --- Copy results back -------------------------------------------------------
diff --git a/scripts/run_bench_collect.sh b/scripts/run_bench_collect.sh
new file mode 100755
index 0000000..75223a3
--- /dev/null
+++ b/scripts/run_bench_collect.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+usage() {
+ cat <<'EOF'
+usage:
+ ./scripts/run_bench_collect.sh
+
+Runs the benchmark suite and appends results to bench/history.jsonl.
+Does NOT update README.md or regenerate chart.svg.
+
+Use run_bench_update.sh to update the chart and README from collected data.
+
+Environment:
+ PARRHESIA_BENCH_RUNS Number of runs (default: 3)
+ PARRHESIA_BENCH_MACHINE_ID Machine identifier (default: hostname -s)
+
+All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded.
+
+Example:
+ # Collect benchmark data
+ ./scripts/run_bench_collect.sh
+
+ # Later, update chart and README
+ ./scripts/run_bench_update.sh
+EOF
+}
+
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+ usage
+ exit 0
+fi
+
+# --- Configuration -----------------------------------------------------------
+
+BENCH_DIR="$ROOT_DIR/bench"
+HISTORY_FILE="$BENCH_DIR/history.jsonl"
+
+MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}"
+GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')"
+GIT_COMMIT="$(git rev-parse --short=7 HEAD)"
+TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+RUNS="${PARRHESIA_BENCH_RUNS:-3}"
+
+mkdir -p "$BENCH_DIR"
+
+WORK_DIR="$(mktemp -d)"
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+JSON_OUT="$WORK_DIR/bench_summary.json"
+RAW_OUTPUT="$WORK_DIR/bench_output.txt"
+
+# --- Phase 1: Run benchmarks -------------------------------------------------
+
+echo "Running ${RUNS}-run benchmark suite..."
+
+PARRHESIA_BENCH_RUNS="$RUNS" \
+BENCH_JSON_OUT="$JSON_OUT" \
+ ./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT"
+
+if [[ ! -f "$JSON_OUT" ]]; then
+ echo "Benchmark JSON output not found at $JSON_OUT" >&2
+ exit 1
+fi
+
+# --- Phase 2: Append to history ----------------------------------------------
+
+echo "Appending to history..."
+
+node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE'
+const fs = require("node:fs");
+
+const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv;
+
+const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
+
+const entry = {
+ timestamp,
+ machine_id: machineId,
+ git_tag: gitTag,
+ git_commit: gitCommit,
+ runs: Number(runsStr),
+ versions: versions || {},
+ servers,
+};
+
+fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8");
+console.log(" entry: " + gitTag + " (" + gitCommit + ") on " + machineId);
+NODE
+
+# --- Done ---------------------------------------------------------------------
+
+echo
+echo "Benchmark data collected and appended to $HISTORY_FILE"
+echo
+echo "To update chart and README with collected data:"
+echo " ./scripts/run_bench_update.sh"
+echo
+echo "To update for a specific machine:"
+echo " ./scripts/run_bench_update.sh "
diff --git a/scripts/run_bench_update.sh b/scripts/run_bench_update.sh
index 06e011f..40e2e54 100755
--- a/scripts/run_bench_update.sh
+++ b/scripts/run_bench_update.sh
@@ -7,18 +7,25 @@ cd "$ROOT_DIR"
usage() {
cat <<'EOF'
usage:
+ ./scripts/run_bench_update.sh [machine_id]
+
+Regenerates bench/chart.svg and updates the benchmark table in README.md
+from collected data in bench/history.jsonl.
+
+Arguments:
+ machine_id Optional. Filter to a specific machine's data.
+ Default: current machine (hostname -s)
+ Use "all" to include all machines (will use latest entry per tag)
+
+Examples:
+ # Update chart for current machine
./scripts/run_bench_update.sh
-Runs the benchmark suite (3 runs by default), then:
- 1) Appends structured results to bench/history.jsonl
- 2) Generates bench/chart.svg via gnuplot
- 3) Updates the comparison table in README.md
+ # Update chart for specific machine
+ ./scripts/run_bench_update.sh my-server
-Environment:
- PARRHESIA_BENCH_RUNS Number of runs (default: 3)
- PARRHESIA_BENCH_MACHINE_ID Machine identifier (default: hostname -s)
-
-All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded.
+ # Update chart using all machines (latest entry per tag wins)
+ ./scripts/run_bench_update.sh all
EOF
}
@@ -34,61 +41,20 @@ HISTORY_FILE="$BENCH_DIR/history.jsonl"
CHART_FILE="$BENCH_DIR/chart.svg"
GNUPLOT_TEMPLATE="$BENCH_DIR/chart.gnuplot"
-MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}"
-GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')"
-GIT_COMMIT="$(git rev-parse --short=7 HEAD)"
-TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-RUNS="${PARRHESIA_BENCH_RUNS:-3}"
+MACHINE_ID="${1:-$(hostname -s)}"
-mkdir -p "$BENCH_DIR"
+if [[ ! -f "$HISTORY_FILE" ]]; then
+ echo "Error: No history file found at $HISTORY_FILE" >&2
+ echo "Run ./scripts/run_bench_collect.sh first to collect benchmark data" >&2
+ exit 1
+fi
WORK_DIR="$(mktemp -d)"
trap 'rm -rf "$WORK_DIR"' EXIT
-JSON_OUT="$WORK_DIR/bench_summary.json"
-RAW_OUTPUT="$WORK_DIR/bench_output.txt"
+# --- Generate chart ----------------------------------------------------------
-# --- Phase 1: Run benchmarks -------------------------------------------------
-
-echo "Running ${RUNS}-run benchmark suite..."
-
-PARRHESIA_BENCH_RUNS="$RUNS" \
-BENCH_JSON_OUT="$JSON_OUT" \
- ./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT"
-
-if [[ ! -f "$JSON_OUT" ]]; then
- echo "Benchmark JSON output not found at $JSON_OUT" >&2
- exit 1
-fi
-
-# --- Phase 2: Append to history ----------------------------------------------
-
-echo "Appending to history..."
-
-node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE'
-const fs = require("node:fs");
-
-const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv;
-
-const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
-
-const entry = {
- timestamp,
- machine_id: machineId,
- git_tag: gitTag,
- git_commit: gitCommit,
- runs: Number(runsStr),
- versions: versions || {},
- servers,
-};
-
-fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8");
-console.log(" entry: " + gitTag + " (" + gitCommit + ") on " + machineId);
-NODE
-
-# --- Phase 3: Generate chart --------------------------------------------------
-
-echo "Generating chart..."
+echo "Generating chart for machine: $MACHINE_ID"
node - "$HISTORY_FILE" "$MACHINE_ID" "$WORK_DIR" <<'NODE'
const fs = require("node:fs");
@@ -106,15 +72,23 @@ const lines = fs.readFileSync(historyFile, "utf8")
.filter(l => l.trim().length > 0)
.map(l => JSON.parse(l));
-// Filter to current machine
-const entries = lines.filter(e => e.machine_id === machineId);
+// Filter to selected machine(s)
+let entries;
+if (machineId === "all") {
+ entries = lines;
+ console.log(" using all machines");
+} else {
+ entries = lines.filter(e => e.machine_id === machineId);
+ console.log(" filtered to machine: " + machineId);
+}
if (entries.length === 0) {
console.log(" no history entries for machine '" + machineId + "', skipping chart");
process.exit(0);
}
-// Sort chronologically, deduplicate by tag (latest wins)
+// Sort chronologically, deduplicate by tag (latest wins),
+// then order the resulting series by git tag.
entries.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
const byTag = new Map();
for (const e of entries) {
@@ -122,6 +96,22 @@ for (const e of entries) {
}
const deduped = [...byTag.values()];
+function parseSemverTag(tag) {
+ const match = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(tag);
+ return match ? match.slice(1).map(Number) : null;
+}
+
+deduped.sort((a, b) => {
+ const aTag = parseSemverTag(a.git_tag);
+ const bTag = parseSemverTag(b.git_tag);
+
+ if (aTag && bTag) {
+ return aTag[0] - bTag[0] || aTag[1] - bTag[1] || aTag[2] - bTag[2];
+ }
+
+ return a.git_tag.localeCompare(b.git_tag, undefined, { numeric: true });
+});
+
// Determine which non-parrhesia servers are present
const baselineServerNames = ["strfry", "nostr-rs-relay"];
const presentBaselines = baselineServerNames.filter(srv =>
@@ -196,25 +186,68 @@ if [[ -f "$WORK_DIR/plot_commands.gnuplot" ]]; then
echo " chart written to $CHART_FILE"
else
echo " chart generation skipped (no data for this machine)"
+ exit 0
fi
-# --- Phase 4: Update README.md -----------------------------------------------
+# --- Update README.md -------------------------------------------------------
-echo "Updating README.md..."
+echo "Updating README.md with latest benchmark..."
-node - "$JSON_OUT" "$ROOT_DIR/README.md" <<'NODE'
+# Find the most recent entry for this machine
+LATEST_ENTRY=$(node - "$HISTORY_FILE" "$MACHINE_ID" <<'NODE'
+const fs = require("node:fs");
+const [, , historyFile, machineId] = process.argv;
+
+const lines = fs.readFileSync(historyFile, "utf8")
+ .split("\n")
+ .filter(l => l.trim().length > 0)
+ .map(l => JSON.parse(l));
+
+let entries;
+if (machineId === "all") {
+ entries = lines;
+} else {
+ entries = lines.filter(e => e.machine_id === machineId);
+}
+
+if (entries.length === 0) {
+ console.error("No entries found for machine: " + machineId);
+ process.exit(1);
+}
+
+// Get latest entry
+entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+console.log(JSON.stringify(entries[0]));
+NODE
+)
+
+if [[ -z "$LATEST_ENTRY" ]]; then
+ echo "Warning: Could not find latest entry, skipping README update" >&2
+ exit 0
+fi
+
+node - "$LATEST_ENTRY" "$ROOT_DIR/README.md" <<'NODE'
const fs = require("node:fs");
-const [, , jsonOut, readmePath] = process.argv;
+const [, , entryJson, readmePath] = process.argv;
-const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
-const readme = fs.readFileSync(readmePath, "utf8");
+const entry = JSON.parse(entryJson);
+const servers = entry.servers || {};
const pg = servers["parrhesia-pg"];
const mem = servers["parrhesia-memory"];
const strfry = servers["strfry"];
const nostrRs = servers["nostr-rs-relay"];
+if (!pg || !mem) {
+ const present = Object.keys(servers).sort().join(", ") || "(none)";
+ console.error(
+ "Latest benchmark entry must include parrhesia-pg and parrhesia-memory. Present servers: " +
+ present
+ );
+ process.exit(1);
+}
+
function toFixed(v, d = 2) {
return Number.isFinite(v) ? v.toFixed(d) : "n/a";
}
@@ -232,14 +265,14 @@ function boldIf(ratioStr, lowerIsBetter) {
}
const metricRows = [
- ["connect avg latency (ms) \u2193", "connect_avg_ms", true],
- ["connect max latency (ms) \u2193", "connect_max_ms", true],
- ["echo throughput (TPS) \u2191", "echo_tps", false],
- ["echo throughput (MiB/s) \u2191", "echo_mibs", false],
- ["event throughput (TPS) \u2191", "event_tps", false],
- ["event throughput (MiB/s) \u2191", "event_mibs", false],
- ["req throughput (TPS) \u2191", "req_tps", false],
- ["req throughput (MiB/s) \u2191", "req_mibs", false],
+ ["connect avg latency (ms) ↓", "connect_avg_ms", true],
+ ["connect max latency (ms) ↓", "connect_max_ms", true],
+ ["echo throughput (TPS) ↑", "echo_tps", false],
+ ["echo throughput (MiB/s) ↑", "echo_mibs", false],
+ ["event throughput (TPS) ↑", "event_tps", false],
+ ["event throughput (MiB/s) ↑", "event_mibs", false],
+ ["req throughput (TPS) ↑", "req_tps", false],
+ ["req throughput (MiB/s) ↑", "req_mibs", false],
];
const hasStrfry = !!strfry;
@@ -275,6 +308,7 @@ const tableLines = [
];
// Replace the first markdown table in the ## Benchmark section
+const readme = fs.readFileSync(readmePath, "utf8");
const readmeLines = readme.split("\n");
const benchIdx = readmeLines.findIndex(l => /^## Benchmark/.test(l));
if (benchIdx === -1) {
@@ -309,8 +343,7 @@ NODE
# --- Done ---------------------------------------------------------------------
echo
-echo "Benchmark update complete. Files changed:"
-echo " $HISTORY_FILE"
+echo "Benchmark rendering complete. Files updated:"
echo " $CHART_FILE"
echo " $ROOT_DIR/README.md"
echo