bench: Split scripts

mix.exs

@@ -26,7 +26,15 @@ defmodule Parrhesia.MixProject do
   defp elixirc_paths(_env), do: ["lib"]
 
   def cli do
-    [preferred_envs: [precommit: :test, bench: :test, "bench.update": :test]]
+    [
+      preferred_envs: [
+        precommit: :test,
+        bench: :test,
+        "bench.collect": :test,
+        "bench.update": :test,
+        "bench.at": :test
+      ]
+    ]
   end
 
   # Run "mix help deps" to learn about dependencies.
@@ -71,7 +79,9 @@ defmodule Parrhesia.MixProject do
       "test.node_sync_e2e": ["cmd ./scripts/run_node_sync_e2e.sh"],
       "test.node_sync_docker_e2e": ["cmd ./scripts/run_node_sync_docker_e2e.sh"],
       bench: ["cmd ./scripts/run_bench_compare.sh"],
+      "bench.collect": ["cmd ./scripts/run_bench_collect.sh"],
       "bench.update": ["cmd ./scripts/run_bench_update.sh"],
+      "bench.at": ["cmd ./scripts/run_bench_at_ref.sh"],
       # cov: ["cmd mix coveralls.lcov"],
       lint: ["format --check-formatted", "credo"],
       precommit: [

scripts/run_bench_at_ref.sh

@@ -67,7 +67,7 @@ cd "$WORKTREE_DIR"
 # Always copy latest benchmark scripts to ensure consistency
 echo "Copying latest benchmark infrastructure from current..."
 mkdir -p scripts bench
-cp "$ROOT_DIR/scripts/run_bench_update.sh" scripts/
+cp "$ROOT_DIR/scripts/run_bench_collect.sh" scripts/
 cp "$ROOT_DIR/scripts/run_bench_compare.sh" scripts/
 cp "$ROOT_DIR/scripts/run_nostr_bench.sh" scripts/
 if [[ -f "$ROOT_DIR/scripts/run_nostr_bench_strfry.sh" ]]; then
@@ -76,9 +76,6 @@ fi
 if [[ -f "$ROOT_DIR/scripts/run_nostr_bench_nostr_rs_relay.sh" ]]; then
   cp "$ROOT_DIR/scripts/run_nostr_bench_nostr_rs_relay.sh" scripts/
 fi
-if [[ -f "$ROOT_DIR/bench/chart.gnuplot" ]]; then
-  cp "$ROOT_DIR/bench/chart.gnuplot" bench/
-fi
 echo
 
 echo "Installing dependencies..."
@@ -90,11 +87,10 @@ echo
 
 RUNS="${PARRHESIA_BENCH_RUNS:-3}"
 
-# Run the benchmark update script which will append to history.jsonl
+# Run the benchmark collect script which will append to history.jsonl
-# Allow it to fail (e.g., README update might fail on old versions) but continue
 PARRHESIA_BENCH_RUNS="$RUNS" \
 PARRHESIA_BENCH_MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-}" \
-  ./scripts/run_bench_update.sh || echo "Benchmark script exited with error (may be expected for old versions)"
+  ./scripts/run_bench_collect.sh
 
 # --- Copy results back -------------------------------------------------------
 

scripts/run_bench_collect.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+usage() {
+  cat <<'EOF'
+usage:
+  ./scripts/run_bench_collect.sh
+
+Runs the benchmark suite and appends results to bench/history.jsonl.
+Does NOT update README.md or regenerate chart.svg.
+
+Use run_bench_update.sh to update the chart and README from collected data.
+
+Environment:
+  PARRHESIA_BENCH_RUNS               Number of runs (default: 3)
+  PARRHESIA_BENCH_MACHINE_ID         Machine identifier (default: hostname -s)
+
+All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded.
+
+Example:
+  # Collect benchmark data
+  ./scripts/run_bench_collect.sh
+
+  # Later, update chart and README
+  ./scripts/run_bench_update.sh
+EOF
+}
+
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+# --- Configuration -----------------------------------------------------------
+
+BENCH_DIR="$ROOT_DIR/bench"
+HISTORY_FILE="$BENCH_DIR/history.jsonl"
+
+MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}"
+GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')"
+GIT_COMMIT="$(git rev-parse --short=7 HEAD)"
+TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+RUNS="${PARRHESIA_BENCH_RUNS:-3}"
+
+mkdir -p "$BENCH_DIR"
+
+WORK_DIR="$(mktemp -d)"
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+JSON_OUT="$WORK_DIR/bench_summary.json"
+RAW_OUTPUT="$WORK_DIR/bench_output.txt"
+
+# --- Phase 1: Run benchmarks -------------------------------------------------
+
+echo "Running ${RUNS}-run benchmark suite..."
+
+PARRHESIA_BENCH_RUNS="$RUNS" \
+BENCH_JSON_OUT="$JSON_OUT" \
+  ./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT"
+
+if [[ ! -f "$JSON_OUT" ]]; then
+  echo "Benchmark JSON output not found at $JSON_OUT" >&2
+  exit 1
+fi
+
+# --- Phase 2: Append to history ----------------------------------------------
+
+echo "Appending to history..."
+
+node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE'
+const fs = require("node:fs");
+
+const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv;
+
+const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
+
+const entry = {
+  timestamp,
+  machine_id: machineId,
+  git_tag: gitTag,
+  git_commit: gitCommit,
+  runs: Number(runsStr),
+  versions: versions || {},
+  servers,
+};
+
+fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8");
+console.log("  entry: " + gitTag + " (" + gitCommit + ") on " + machineId);
+NODE
+
+# --- Done ---------------------------------------------------------------------
+
+echo
+echo "Benchmark data collected and appended to $HISTORY_FILE"
+echo
+echo "To update chart and README with collected data:"
+echo "  ./scripts/run_bench_update.sh"
+echo
+echo "To update for a specific machine:"
+echo "  ./scripts/run_bench_update.sh <machine_id>"

scripts/run_bench_update.sh

@@ -7,18 +7,25 @@ cd "$ROOT_DIR"
 usage() {
   cat <<'EOF'
 usage:
+  ./scripts/run_bench_update.sh [machine_id]
+
+Regenerates bench/chart.svg and updates the benchmark table in README.md
+from collected data in bench/history.jsonl.
+
+Arguments:
+  machine_id    Optional. Filter to a specific machine's data.
+                Default: current machine (hostname -s)
+                Use "all" to include all machines (will use latest entry per tag)
+
+Examples:
+  # Update chart for current machine
   ./scripts/run_bench_update.sh
 
-Runs the benchmark suite (3 runs by default), then:
+  # Update chart for specific machine
-  1) Appends structured results to bench/history.jsonl
+  ./scripts/run_bench_update.sh my-server
-  2) Generates bench/chart.svg via gnuplot
-  3) Updates the comparison table in README.md
 
-Environment:
+  # Update chart using all machines (latest entry per tag wins)
-  PARRHESIA_BENCH_RUNS               Number of runs (default: 3)
+  ./scripts/run_bench_update.sh all
-  PARRHESIA_BENCH_MACHINE_ID         Machine identifier (default: hostname -s)
-
-All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded.
 EOF
 }
 
@@ -34,61 +41,20 @@ HISTORY_FILE="$BENCH_DIR/history.jsonl"
 CHART_FILE="$BENCH_DIR/chart.svg"
 GNUPLOT_TEMPLATE="$BENCH_DIR/chart.gnuplot"
 
-MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}"
+MACHINE_ID="${1:-$(hostname -s)}"
-GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')"
-GIT_COMMIT="$(git rev-parse --short=7 HEAD)"
-TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-RUNS="${PARRHESIA_BENCH_RUNS:-3}"
 
-mkdir -p "$BENCH_DIR"
+if [[ ! -f "$HISTORY_FILE" ]]; then
+  echo "Error: No history file found at $HISTORY_FILE" >&2
+  echo "Run ./scripts/run_bench_collect.sh first to collect benchmark data" >&2
+  exit 1
+fi
 
 WORK_DIR="$(mktemp -d)"
 trap 'rm -rf "$WORK_DIR"' EXIT
 
-JSON_OUT="$WORK_DIR/bench_summary.json"
+# --- Generate chart ----------------------------------------------------------
-RAW_OUTPUT="$WORK_DIR/bench_output.txt"
 
-# --- Phase 1: Run benchmarks -------------------------------------------------
+echo "Generating chart for machine: $MACHINE_ID"
-
-echo "Running ${RUNS}-run benchmark suite..."
-
-PARRHESIA_BENCH_RUNS="$RUNS" \
-BENCH_JSON_OUT="$JSON_OUT" \
-  ./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT"
-
-if [[ ! -f "$JSON_OUT" ]]; then
-  echo "Benchmark JSON output not found at $JSON_OUT" >&2
-  exit 1
-fi
-
-# --- Phase 2: Append to history ----------------------------------------------
-
-echo "Appending to history..."
-
-node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE'
-const fs = require("node:fs");
-
-const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv;
-
-const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
-
-const entry = {
-  timestamp,
-  machine_id: machineId,
-  git_tag: gitTag,
-  git_commit: gitCommit,
-  runs: Number(runsStr),
-  versions: versions || {},
-  servers,
-};
-
-fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8");
-console.log("  entry: " + gitTag + " (" + gitCommit + ") on " + machineId);
-NODE
-
-# --- Phase 3: Generate chart --------------------------------------------------
-
-echo "Generating chart..."
 
 node - "$HISTORY_FILE" "$MACHINE_ID" "$WORK_DIR" <<'NODE'
 const fs = require("node:fs");
@@ -106,8 +72,15 @@ const lines = fs.readFileSync(historyFile, "utf8")
   .filter(l => l.trim().length > 0)
   .map(l => JSON.parse(l));
 
-// Filter to current machine
+// Filter to selected machine(s)
-const entries = lines.filter(e => e.machine_id === machineId);
+let entries;
+if (machineId === "all") {
+  entries = lines;
+  console.log("  using all machines");
+} else {
+  entries = lines.filter(e => e.machine_id === machineId);
+  console.log("  filtered to machine: " + machineId);
+}
 
 if (entries.length === 0) {
   console.log("  no history entries for machine '" + machineId + "', skipping chart");
@@ -196,19 +169,53 @@ if [[ -f "$WORK_DIR/plot_commands.gnuplot" ]]; then
   echo "  chart written to $CHART_FILE"
 else
   echo "  chart generation skipped (no data for this machine)"
+  exit 0
 fi
 
-# --- Phase 4: Update README.md -----------------------------------------------
+# --- Update README.md -------------------------------------------------------
 
-echo "Updating README.md..."
+echo "Updating README.md with latest benchmark..."
 
-node - "$JSON_OUT" "$ROOT_DIR/README.md" <<'NODE'
+# Find the most recent entry for this machine
+LATEST_ENTRY=$(node - "$HISTORY_FILE" "$MACHINE_ID" <<'NODE'
+const fs = require("node:fs");
+const [, , historyFile, machineId] = process.argv;
+
+const lines = fs.readFileSync(historyFile, "utf8")
+  .split("\n")
+  .filter(l => l.trim().length > 0)
+  .map(l => JSON.parse(l));
+
+let entries;
+if (machineId === "all") {
+  entries = lines;
+} else {
+  entries = lines.filter(e => e.machine_id === machineId);
+}
+
+if (entries.length === 0) {
+  console.error("No entries found for machine: " + machineId);
+  process.exit(1);
+}
+
+// Get latest entry
+entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+console.log(JSON.stringify(entries[0]));
+NODE
+)
+
+if [[ -z "$LATEST_ENTRY" ]]; then
+  echo "Warning: Could not find latest entry, skipping README update" >&2
+  exit 0
+fi
+
+node - "$LATEST_ENTRY" "$ROOT_DIR/README.md" <<'NODE'
 const fs = require("node:fs");
 
-const [, , jsonOut, readmePath] = process.argv;
+const [, , entryJson, readmePath] = process.argv;
 
-const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
+const entry = JSON.parse(entryJson);
-const readme = fs.readFileSync(readmePath, "utf8");
+const { versions, ...servers } = entry;
 
 const pg = servers["parrhesia-pg"];
 const mem = servers["parrhesia-memory"];
@@ -232,14 +239,14 @@ function boldIf(ratioStr, lowerIsBetter) {
 }
 
 const metricRows = [
-  ["connect avg latency (ms) \u2193", "connect_avg_ms", true],
+  ["connect avg latency (ms) ↓", "connect_avg_ms", true],
-  ["connect max latency (ms) \u2193", "connect_max_ms", true],
+  ["connect max latency (ms) ↓", "connect_max_ms", true],
-  ["echo throughput (TPS) \u2191",    "echo_tps",       false],
+  ["echo throughput (TPS) ↑",    "echo_tps",       false],
-  ["echo throughput (MiB/s) \u2191",  "echo_mibs",      false],
+  ["echo throughput (MiB/s) ↑",  "echo_mibs",      false],
-  ["event throughput (TPS) \u2191",   "event_tps",      false],
+  ["event throughput (TPS) ↑",   "event_tps",      false],
-  ["event throughput (MiB/s) \u2191", "event_mibs",     false],
+  ["event throughput (MiB/s) ↑", "event_mibs",     false],
-  ["req throughput (TPS) \u2191",     "req_tps",        false],
+  ["req throughput (TPS) ↑",     "req_tps",        false],
-  ["req throughput (MiB/s) \u2191",   "req_mibs",       false],
+  ["req throughput (MiB/s) ↑",   "req_mibs",       false],
 ];
 
 const hasStrfry = !!strfry;
@@ -275,6 +282,7 @@ const tableLines = [
 ];
 
 // Replace the first markdown table in the ## Benchmark section
+const readme = fs.readFileSync(readmePath, "utf8");
 const readmeLines = readme.split("\n");
 const benchIdx = readmeLines.findIndex(l => /^## Benchmark/.test(l));
 if (benchIdx === -1) {
@@ -309,8 +317,7 @@ NODE
 # --- Done ---------------------------------------------------------------------
 
 echo
-echo "Benchmark update complete. Files changed:"
+echo "Benchmark rendering complete. Files updated:"
-echo "  $HISTORY_FILE"
 echo "  $CHART_FILE"
 echo "  $ROOT_DIR/README.md"
 echo