diff --git a/bench/history.jsonl b/bench/history.jsonl deleted file mode 100644 index 98dac89..0000000 --- a/bench/history.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"timestamp":"2026-03-18T20:13:21Z","machine_id":"squirrel","git_tag":"v0.5.0","git_commit":"970cee2","runs":3,"servers":{"parrhesia-pg":{"connect_avg_ms":9.333333333333334,"connect_max_ms":12.333333333333334,"echo_tps":64030.333333333336,"echo_mibs":35.06666666666666,"event_tps":5015.333333333333,"event_mibs":3.4,"req_tps":6416.333333333333,"req_mibs":42.43333333333334},"parrhesia-memory":{"connect_avg_ms":7.666666666666667,"connect_max_ms":9.666666666666666,"echo_tps":93656.33333333333,"echo_mibs":51.26666666666667,"event_tps":1505.3333333333333,"event_mibs":1,"req_tps":14566.666666666666,"req_mibs":94.23333333333335},"nostr-rs-relay":{"connect_avg_ms":7,"connect_max_ms":10.333333333333334,"echo_tps":140767,"echo_mibs":77.06666666666666,"event_tps":2293.6666666666665,"event_mibs":1.5,"req_tps":3035.6666666666665,"req_mibs":19.23333333333333}}} diff --git a/scripts/run_bench_compare.sh b/scripts/run_bench_compare.sh index 094bef4..f0e15c9 100755 --- a/scripts/run_bench_compare.sh +++ b/scripts/run_bench_compare.sh @@ -298,7 +298,8 @@ for run in $(seq 1 "$RUNS"); do done -node - "$WORK_DIR" "$RUNS" "$HAS_STRFRY" "$HAS_NOSTR_RS" <<'NODE' +node - "$WORK_DIR" "$RUNS" "$HAS_STRFRY" "$HAS_NOSTR_RS" \ + "$PARRHESIA_VERSION" "$STRFRY_VERSION" "$NOSTR_RS_RELAY_VERSION" "$NOSTR_BENCH_VERSION" <<'NODE' const fs = require("node:fs"); const path = require("node:path"); @@ -306,6 +307,10 @@ const workDir = process.argv[2]; const runs = Number(process.argv[3]); const hasStrfry = process.argv[4] === "1"; const hasNostrRs = process.argv[5] === "1"; +const parrhesiaVersion = process.argv[6] || ""; +const strfryVersion = process.argv[7] || ""; +const nostrRsRelayVersion = process.argv[8] || ""; +const nostrBenchVersion = process.argv[9] || ""; function parseLog(filePath) { const content = fs.readFileSync(filePath, "utf8"); @@ -502,6 +507,12 @@ if (process.env.BENCH_JSON_OUT) { }; } + const versions = { parrhesia: parrhesiaVersion }; + if (hasStrfry && strfryVersion) versions.strfry = strfryVersion; + if (hasNostrRs && nostrRsRelayVersion) versions["nostr-rs-relay"] = nostrRsRelayVersion; + if (nostrBenchVersion) versions["nostr-bench"] = nostrBenchVersion; + jsonSummary.versions = versions; + fs.writeFileSync( process.env.BENCH_JSON_OUT, JSON.stringify(jsonSummary, null, 2) + "\n", diff --git a/scripts/run_bench_update.sh b/scripts/run_bench_update.sh index 7fa06d3..06e011f 100755 --- a/scripts/run_bench_update.sh +++ b/scripts/run_bench_update.sh @@ -70,7 +70,7 @@ const fs = require("node:fs"); const [, , jsonOut, timestamp, machineId, gitTag, gitCommit, runsStr, historyFile] = process.argv; -const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8")); +const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8")); const entry = { timestamp, @@ -78,6 +78,7 @@ const entry = { git_tag: gitTag, git_commit: gitCommit, runs: Number(runsStr), + versions: versions || {}, servers, }; @@ -127,19 +128,6 @@ const presentBaselines = baselineServerNames.filter(srv => deduped.some(e => e.servers[srv]) ); -// Compute averages for baseline servers (constant horizontal lines) -const baselineAvg = {}; -for (const srv of presentBaselines) { - const vals = deduped.filter(e => e.servers[srv]).map(e => e.servers[srv]); - baselineAvg[srv] = {}; - for (const metric of Object.keys(vals[0])) { - const valid = vals.map(v => v[metric]).filter(Number.isFinite); - baselineAvg[srv][metric] = valid.length > 0 - ? valid.reduce((a, b) => a + b, 0) / valid.length - : NaN; - } -} - // Metrics to chart const chartMetrics = [ { key: "event_tps", label: "Event Throughput (TPS) — higher is better", file: "event_tps.tsv", ylabel: "TPS" }, @@ -161,7 +149,7 @@ for (const cm of chartMetrics) { e.servers["parrhesia-memory"]?.[cm.key] ?? "NaN", ]; for (const srv of presentBaselines) { - row.push(baselineAvg[srv]?.[cm.key] ?? "NaN"); + row.push(e.servers[srv]?.[cm.key] ?? "NaN"); } rows.push(row.join("\t")); } @@ -171,7 +159,7 @@ for (const cm of chartMetrics) { // Generate gnuplot plot commands (handles variable column counts) const serverLabels = ["parrhesia-pg", "parrhesia-memory"]; -for (const srv of presentBaselines) serverLabels.push(srv + " (avg)"); +for (const srv of presentBaselines) serverLabels.push(srv); const plotLines = []; for (const cm of chartMetrics) { @@ -219,7 +207,7 @@ const fs = require("node:fs"); const [, , jsonOut, readmePath] = process.argv; -const servers = JSON.parse(fs.readFileSync(jsonOut, "utf8")); +const { versions, ...servers } = JSON.parse(fs.readFileSync(jsonOut, "utf8")); const readme = fs.readFileSync(readmePath, "utf8"); const pg = servers["parrhesia-pg"];