From a2fd632cc4ea030859c8541edd13b19e1d0fe0af Mon Sep 17 00:00:00 2001 From: Steffen Beyer Date: Thu, 19 Mar 2026 00:12:20 +0100 Subject: [PATCH] bench: Split scripts --- mix.exs | 12 ++- scripts/run_bench_at_ref.sh | 10 +-- scripts/run_bench_collect.sh | 103 +++++++++++++++++++++++ scripts/run_bench_update.sh | 159 ++++++++++++++++++----------------- 4 files changed, 200 insertions(+), 84 deletions(-) create mode 100755 scripts/run_bench_collect.sh diff --git a/mix.exs b/mix.exs index 458816d..c1d655a 100644 --- a/mix.exs +++ b/mix.exs @@ -26,7 +26,15 @@ defmodule Parrhesia.MixProject do defp elixirc_paths(_env), do: ["lib"] def cli do - [preferred_envs: [precommit: :test, bench: :test, "bench.update": :test]] + [ + preferred_envs: [ + precommit: :test, + bench: :test, + "bench.collect": :test, + "bench.update": :test, + "bench.at": :test + ] + ] end # Run "mix help deps" to learn about dependencies. @@ -71,7 +79,9 @@ defmodule Parrhesia.MixProject do "test.node_sync_e2e": ["cmd ./scripts/run_node_sync_e2e.sh"], "test.node_sync_docker_e2e": ["cmd ./scripts/run_node_sync_docker_e2e.sh"], bench: ["cmd ./scripts/run_bench_compare.sh"], + "bench.collect": ["cmd ./scripts/run_bench_collect.sh"], "bench.update": ["cmd ./scripts/run_bench_update.sh"], + "bench.at": ["cmd ./scripts/run_bench_at_ref.sh"], # cov: ["cmd mix coveralls.lcov"], lint: ["format --check-formatted", "credo"], precommit: [ diff --git a/scripts/run_bench_at_ref.sh b/scripts/run_bench_at_ref.sh index 0db8d20..12e7e81 100755 --- a/scripts/run_bench_at_ref.sh +++ b/scripts/run_bench_at_ref.sh @@ -67,7 +67,7 @@ cd "$WORKTREE_DIR" # Always copy latest benchmark scripts to ensure consistency echo "Copying latest benchmark infrastructure from current..." mkdir -p scripts bench -cp "$ROOT_DIR/scripts/run_bench_update.sh" scripts/ +cp "$ROOT_DIR/scripts/run_bench_collect.sh" scripts/ cp "$ROOT_DIR/scripts/run_bench_compare.sh" scripts/ cp "$ROOT_DIR/scripts/run_nostr_bench.sh" scripts/ if [[ -f "$ROOT_DIR/scripts/run_nostr_bench_strfry.sh" ]]; then @@ -76,9 +76,6 @@ fi if [[ -f "$ROOT_DIR/scripts/run_nostr_bench_nostr_rs_relay.sh" ]]; then cp "$ROOT_DIR/scripts/run_nostr_bench_nostr_rs_relay.sh" scripts/ fi -if [[ -f "$ROOT_DIR/bench/chart.gnuplot" ]]; then - cp "$ROOT_DIR/bench/chart.gnuplot" bench/ -fi echo echo "Installing dependencies..." @@ -90,11 +87,10 @@ echo RUNS="${PARRHESIA_BENCH_RUNS:-3}" -# Run the benchmark update script which will append to history.jsonl -# Allow it to fail (e.g., README update might fail on old versions) but continue +# Run the benchmark collect script which will append to history.jsonl PARRHESIA_BENCH_RUNS="$RUNS" \ PARRHESIA_BENCH_MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-}" \ - ./scripts/run_bench_update.sh || echo "Benchmark script exited with error (may be expected for old versions)" + ./scripts/run_bench_collect.sh # --- Copy results back ------------------------------------------------------- diff --git a/scripts/run_bench_collect.sh b/scripts/run_bench_collect.sh new file mode 100755 index 0000000..75223a3 --- /dev/null +++ b/scripts/run_bench_collect.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +usage() { + cat <<'EOF' +usage: + ./scripts/run_bench_collect.sh + +Runs the benchmark suite and appends results to bench/history.jsonl. +Does NOT update README.md or regenerate chart.svg. + +Use run_bench_update.sh to update the chart and README from collected data. + +Environment: + PARRHESIA_BENCH_RUNS Number of runs (default: 3) + PARRHESIA_BENCH_MACHINE_ID Machine identifier (default: hostname -s) + +All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded. + +Example: + # Collect benchmark data + ./scripts/run_bench_collect.sh + + # Later, update chart and README + ./scripts/run_bench_update.sh +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +# --- Configuration ----------------------------------------------------------- + +BENCH_DIR="$ROOT_DIR/bench" +HISTORY_FILE="$BENCH_DIR/history.jsonl" + +MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}" +GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')" +GIT_COMMIT="$(git rev-parse --short=7 HEAD)" +TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +RUNS="${PARRHESIA_BENCH_RUNS:-3}" + +mkdir -p "$BENCH_DIR" + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +JSON_OUT="$WORK_DIR/bench_summary.json" +RAW_OUTPUT="$WORK_DIR/bench_output.txt" + +# --- Phase 1: Run benchmarks ------------------------------------------------- + +echo "Running ${RUNS}-run benchmark suite..." + +PARRHESIA_BENCH_RUNS="$RUNS" \ +BENCH_JSON_OUT="$JSON_OUT" \ + ./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT" + +if [[ ! -f "$JSON_OUT" ]]; then + echo "Benchmark JSON output not found at $JSON_OUT" >&2 + exit 1 +fi + +# --- Phase 2: Append to history ---------------------------------------------- + +echo "Appending to history..." + +node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE' +const fs = require("node:fs"); + +const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv; + +const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8")); + +const entry = { + timestamp, + machine_id: machineId, + git_tag: gitTag, + git_commit: gitCommit, + runs: Number(runsStr), + versions: versions || {}, + servers, +}; + +fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8"); +console.log(" entry: " + gitTag + " (" + gitCommit + ") on " + machineId); +NODE + +# --- Done --------------------------------------------------------------------- + +echo +echo "Benchmark data collected and appended to $HISTORY_FILE" +echo +echo "To update chart and README with collected data:" +echo " ./scripts/run_bench_update.sh" +echo +echo "To update for a specific machine:" +echo " ./scripts/run_bench_update.sh " diff --git a/scripts/run_bench_update.sh b/scripts/run_bench_update.sh index 06e011f..516a024 100755 --- a/scripts/run_bench_update.sh +++ b/scripts/run_bench_update.sh @@ -7,18 +7,25 @@ cd "$ROOT_DIR" usage() { cat <<'EOF' usage: + ./scripts/run_bench_update.sh [machine_id] + +Regenerates bench/chart.svg and updates the benchmark table in README.md +from collected data in bench/history.jsonl. + +Arguments: + machine_id Optional. Filter to a specific machine's data. + Default: current machine (hostname -s) + Use "all" to include all machines (will use latest entry per tag) + +Examples: + # Update chart for current machine ./scripts/run_bench_update.sh -Runs the benchmark suite (3 runs by default), then: - 1) Appends structured results to bench/history.jsonl - 2) Generates bench/chart.svg via gnuplot - 3) Updates the comparison table in README.md + # Update chart for specific machine + ./scripts/run_bench_update.sh my-server -Environment: - PARRHESIA_BENCH_RUNS Number of runs (default: 3) - PARRHESIA_BENCH_MACHINE_ID Machine identifier (default: hostname -s) - -All PARRHESIA_BENCH_* knobs from run_bench_compare.sh are forwarded. + # Update chart using all machines (latest entry per tag wins) + ./scripts/run_bench_update.sh all EOF } @@ -34,61 +41,20 @@ HISTORY_FILE="$BENCH_DIR/history.jsonl" CHART_FILE="$BENCH_DIR/chart.svg" GNUPLOT_TEMPLATE="$BENCH_DIR/chart.gnuplot" -MACHINE_ID="${PARRHESIA_BENCH_MACHINE_ID:-$(hostname -s)}" -GIT_TAG="$(git describe --tags --abbrev=0 2>/dev/null || echo 'untagged')" -GIT_COMMIT="$(git rev-parse --short=7 HEAD)" -TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -RUNS="${PARRHESIA_BENCH_RUNS:-3}" +MACHINE_ID="${1:-$(hostname -s)}" -mkdir -p "$BENCH_DIR" +if [[ ! -f "$HISTORY_FILE" ]]; then + echo "Error: No history file found at $HISTORY_FILE" >&2 + echo "Run ./scripts/run_bench_collect.sh first to collect benchmark data" >&2 + exit 1 +fi WORK_DIR="$(mktemp -d)" trap 'rm -rf "$WORK_DIR"' EXIT -JSON_OUT="$WORK_DIR/bench_summary.json" -RAW_OUTPUT="$WORK_DIR/bench_output.txt" +# --- Generate chart ---------------------------------------------------------- -# --- Phase 1: Run benchmarks ------------------------------------------------- - -echo "Running ${RUNS}-run benchmark suite..." - -PARRHESIA_BENCH_RUNS="$RUNS" \ -BENCH_JSON_OUT="$JSON_OUT" \ - ./scripts/run_bench_compare.sh 2>&1 | tee "$RAW_OUTPUT" - -if [[ ! -f "$JSON_OUT" ]]; then - echo "Benchmark JSON output not found at $JSON_OUT" >&2 - exit 1 -fi - -# --- Phase 2: Append to history ---------------------------------------------- - -echo "Appending to history..." - -node - "$JSON_OUT" "$TIMESTAMP" "$MACHINE_ID" "$GIT_TAG" "$GIT_COMMIT" "$RUNS" "$HISTORY_FILE" <<'NODE' -const fs = require("node:fs"); - -const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv; - -const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8")); - -const entry = { - timestamp, - machine_id: machineId, - git_tag: gitTag, - git_commit: gitCommit, - runs: Number(runsStr), - versions: versions || {}, - servers, -}; - -fs.appendFileSync(historyFile, JSON.stringify(entry) + "\n", "utf8"); -console.log(" entry: " + gitTag + " (" + gitCommit + ") on " + machineId); -NODE - -# --- Phase 3: Generate chart -------------------------------------------------- - -echo "Generating chart..." +echo "Generating chart for machine: $MACHINE_ID" node - "$HISTORY_FILE" "$MACHINE_ID" "$WORK_DIR" <<'NODE' const fs = require("node:fs"); @@ -106,8 +72,15 @@ const lines = fs.readFileSync(historyFile, "utf8") .filter(l => l.trim().length > 0) .map(l => JSON.parse(l)); -// Filter to current machine -const entries = lines.filter(e => e.machine_id === machineId); +// Filter to selected machine(s) +let entries; +if (machineId === "all") { + entries = lines; + console.log(" using all machines"); +} else { + entries = lines.filter(e => e.machine_id === machineId); + console.log(" filtered to machine: " + machineId); +} if (entries.length === 0) { console.log(" no history entries for machine '" + machineId + "', skipping chart"); @@ -196,19 +169,53 @@ if [[ -f "$WORK_DIR/plot_commands.gnuplot" ]]; then echo " chart written to $CHART_FILE" else echo " chart generation skipped (no data for this machine)" + exit 0 fi -# --- Phase 4: Update README.md ----------------------------------------------- +# --- Update README.md ------------------------------------------------------- -echo "Updating README.md..." +echo "Updating README.md with latest benchmark..." -node - "$JSON_OUT" "$ROOT_DIR/README.md" <<'NODE' +# Find the most recent entry for this machine +LATEST_ENTRY=$(node - "$HISTORY_FILE" "$MACHINE_ID" <<'NODE' +const fs = require("node:fs"); +const [, , historyFile, machineId] = process.argv; + +const lines = fs.readFileSync(historyFile, "utf8") + .split("\n") + .filter(l => l.trim().length > 0) + .map(l => JSON.parse(l)); + +let entries; +if (machineId === "all") { + entries = lines; +} else { + entries = lines.filter(e => e.machine_id === machineId); +} + +if (entries.length === 0) { + console.error("No entries found for machine: " + machineId); + process.exit(1); +} + +// Get latest entry +entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp)); +console.log(JSON.stringify(entries[0])); +NODE +) + +if [[ -z "$LATEST_ENTRY" ]]; then + echo "Warning: Could not find latest entry, skipping README update" >&2 + exit 0 +fi + +node - "$LATEST_ENTRY" "$ROOT_DIR/README.md" <<'NODE' const fs = require("node:fs"); -const [, , jsonOut, readmePath] = process.argv; +const [, , entryJson, readmePath] = process.argv; -const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8")); -const readme = fs.readFileSync(readmePath, "utf8"); +const entry = JSON.parse(entryJson); +const { versions, ...servers } = entry; const pg = servers["parrhesia-pg"]; const mem = servers["parrhesia-memory"]; @@ -232,14 +239,14 @@ function boldIf(ratioStr, lowerIsBetter) { } const metricRows = [ - ["connect avg latency (ms) \u2193", "connect_avg_ms", true], - ["connect max latency (ms) \u2193", "connect_max_ms", true], - ["echo throughput (TPS) \u2191", "echo_tps", false], - ["echo throughput (MiB/s) \u2191", "echo_mibs", false], - ["event throughput (TPS) \u2191", "event_tps", false], - ["event throughput (MiB/s) \u2191", "event_mibs", false], - ["req throughput (TPS) \u2191", "req_tps", false], - ["req throughput (MiB/s) \u2191", "req_mibs", false], + ["connect avg latency (ms) ↓", "connect_avg_ms", true], + ["connect max latency (ms) ↓", "connect_max_ms", true], + ["echo throughput (TPS) ↑", "echo_tps", false], + ["echo throughput (MiB/s) ↑", "echo_mibs", false], + ["event throughput (TPS) ↑", "event_tps", false], + ["event throughput (MiB/s) ↑", "event_mibs", false], + ["req throughput (TPS) ↑", "req_tps", false], + ["req throughput (MiB/s) ↑", "req_mibs", false], ]; const hasStrfry = !!strfry; @@ -275,6 +282,7 @@ const tableLines = [ ]; // Replace the first markdown table in the ## Benchmark section +const readme = fs.readFileSync(readmePath, "utf8"); const readmeLines = readme.split("\n"); const benchIdx = readmeLines.findIndex(l => /^## Benchmark/.test(l)); if (benchIdx === -1) { @@ -309,8 +317,7 @@ NODE # --- Done --------------------------------------------------------------------- echo -echo "Benchmark update complete. Files changed:" -echo " $HISTORY_FILE" +echo "Benchmark rendering complete. Files updated:" echo " $CHART_FILE" echo " $ROOT_DIR/README.md" echo