bench: simplify cloud bench flow and align phased naming

scripts/cloud_bench_client.sh

@@ -43,6 +43,9 @@ run_event() {
     -r "${PARRHESIA_BENCH_EVENT_RATE:-50}" \
     -k "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \
     -t "${bench_threads}" \
+    --send-strategy "${PARRHESIA_BENCH_EVENT_SEND_STRATEGY:-pipelined}" \
+    --inflight "${PARRHESIA_BENCH_EVENT_INFLIGHT:-32}" \
+    --ack-timeout "${PARRHESIA_BENCH_EVENT_ACK_TIMEOUT:-30}" \
     "${relay_url}"
 }
 
@@ -68,11 +71,11 @@ run_seed() {
   echo "==> nostr-bench seed ${relay_url}"
   "$bench_bin" seed --json \
     --target-accepted "$target_accepted" \
-    -c "${PARRHESIA_BENCH_SEED_CONNECTION_COUNT:-64}" \
+    -c "${PARRHESIA_BENCH_SEED_CONNECTION_COUNT:-5000}" \
-    -r "${PARRHESIA_BENCH_SEED_CONNECTION_RATE:-64}" \
+    -r "${PARRHESIA_BENCH_SEED_CONNECTION_RATE:-5000}" \
     -k "${PARRHESIA_BENCH_SEED_KEEPALIVE_SECONDS:-0}" \
     -t "${bench_threads}" \
-    --send-strategy "${PARRHESIA_BENCH_SEED_SEND_STRATEGY:-ack-loop}" \
+    --send-strategy "${PARRHESIA_BENCH_SEED_SEND_STRATEGY:-pipelined}" \
     --inflight "${PARRHESIA_BENCH_SEED_INFLIGHT:-32}" \
     --ack-timeout "${PARRHESIA_BENCH_SEED_ACK_TIMEOUT:-30}" \
     "${relay_url}"

scripts/cloud_bench_orchestrate.mjs

@@ -40,10 +40,12 @@ const NOSTREAM_REDIS_IMAGE = "redis:7.0.5-alpine3.16";
 const SEED_TOLERANCE_RATIO = 0.01;
 const SEED_MAX_ROUNDS = 4;
 const SEED_KEEPALIVE_SECONDS = 0;
-const SEED_EVENTS_PER_CONNECTION_TARGET = 2000;
+const SEED_CONNECTION_COUNT = 5000;
-const SEED_CONNECTIONS_MIN = 1;
+const SEED_CONNECTION_RATE = 5000;
-const SEED_CONNECTIONS_MAX = 512;
+const EVENT_SEND_STRATEGY = "pipelined";
-const SEED_SEND_STRATEGY = "ack-loop";
+const EVENT_INFLIGHT = 32;
+const EVENT_ACK_TIMEOUT_SECONDS = 30;
+const SEED_SEND_STRATEGY = "pipelined";
 const SEED_INFLIGHT = 32;
 const SEED_ACK_TIMEOUT_SECONDS = 30;
 const PHASE_PREP_OFFSET_MINUTES = 3;
@@ -54,7 +56,6 @@ const DEFAULTS = {
   clientType: "cpx31",
   imageBase: "ubuntu-24.04",
   clients: 3,
-  runs: 5,
   targets: DEFAULT_TARGETS,
   historyFile: "bench/history.jsonl",
   artifactsDir: "bench/cloud_artifacts",
@@ -73,17 +74,17 @@ const DEFAULTS = {
   warmEvents: 50000,
   hotEvents: 500000,
   bench: {
-    connectCount: 3000,
+    connectCount: 50000,
-    connectRate: 1500,
+    connectRate: 10000,
-    echoCount: 3000,
+    echoCount: 50000,
-    echoRate: 1500,
+    echoRate: 10000,
     echoSize: 512,
-    eventCount: 5000,
+    eventCount: 50000,
-    eventRate: 2000,
+    eventRate: 10000,
-    reqCount: 3000,
+    reqCount: 50000,
-    reqRate: 1500,
+    reqRate: 10000,
     reqLimit: 50,
-    keepaliveSeconds: 10,
+    keepaliveSeconds: 120,
     threads: 0,
   },
 };
@@ -103,7 +104,6 @@ Options:
   --client-type <name>         (default: ${DEFAULTS.clientType})
   --image-base <name>          (default: ${DEFAULTS.imageBase})
   --clients <n>                (default: ${DEFAULTS.clients})
-  --runs <n>                   (default: ${DEFAULTS.runs})
   --targets <csv>              (default: ${DEFAULT_TARGETS.join(",")})
 
   Source selection (choose one style):
@@ -151,7 +151,7 @@ Notes:
   - In interactive terminals, prompts you to pick + confirm the datacenter unless --yes is set.
   - Caches built nostr-bench at _build/bench/nostr-bench and reuses it when valid.
   - Auto-tunes Postgres/Redis/app pool sizing from server RAM + CPU for DB-backed targets.
-  - Randomizes target order per run and wipes persisted target data directories on each start.
+  - Randomizes target order and wipes persisted target data directories on each start.
   - Creates a Hetzner Cloud firewall restricting inbound access to benchmark ports from known IPs only.
   - Handles Ctrl-C / SIGTERM with best-effort cloud cleanup.
   - Tries nix .#nostrBenchStaticX86_64Musl first; falls back to docker-built portable nostr-bench.
@@ -201,9 +201,6 @@ function parseArgs(argv) {
       case "--clients":
         opts.clients = intOpt(arg, argv[++i]);
         break;
-      case "--runs":
-        opts.runs = intOpt(arg, argv[++i]);
-        break;
       case "--targets":
         opts.targets = argv[++i]
           .split(",")
@@ -1053,11 +1050,8 @@ async function runClientSeedingRound({
         };
       }
 
-      const seedConnectionCount = Math.min(
+      const seedConnectionCount = SEED_CONNECTION_COUNT;
-        SEED_CONNECTIONS_MAX,
+      const seedConnectionRate = SEED_CONNECTION_RATE;
-        Math.max(SEED_CONNECTIONS_MIN, Math.ceil(desiredAccepted / SEED_EVENTS_PER_CONNECTION_TARGET)),
-      );
-      const seedConnectionRate = Math.max(1, seedConnectionCount);
 
       const seedEnvPrefix = [
         `PARRHESIA_BENCH_SEED_TARGET_ACCEPTED=${desiredAccepted}`,
@@ -1947,17 +1941,14 @@ async function main() {
     };
 
     const results = [];
-    const targetOrderPerRun = [];
+    const targetOrder = shuffled(opts.targets);
 
     phaseLogger.logPhase(`[phase] benchmark execution (mode=${opts.quick ? "quick" : "phased"})`);
 
-    for (let runIndex = 1; runIndex <= opts.runs; runIndex += 1) {
+    console.log(`[bench] target-order=${targetOrder.join(",")}`);
-      const runTargets = shuffled(opts.targets);
-      targetOrderPerRun.push({ run: runIndex, targets: runTargets });
-      console.log(`[bench] run ${runIndex}/${opts.runs} target-order=${runTargets.join(",")}`);
 
-      for (const target of runTargets) {
+    for (const target of targetOrder) {
-        console.log(`[bench] run ${runIndex}/${opts.runs} target=${target}`);
+      console.log(`[bench] target=${target}`);
       const targetStartTime = new Date().toISOString();
 
       const serverEnvPrefix = [
@@ -1975,7 +1966,7 @@ async function main() {
       try {
         await sshExec(serverIp, keyPath, `${serverEnvPrefix} /root/cloud-bench-server.sh ${shellEscape(startCommands[target])}`);
       } catch (error) {
-          console.error(`[bench] target startup failed target=${target} run=${runIndex}`);
+        console.error(`[bench] target startup failed target=${target}`);
         if (error?.stdout?.trim()) {
           console.error(`[bench] server startup stdout:\n${error.stdout.trim()}`);
         }
@@ -1986,7 +1977,7 @@ async function main() {
       }
 
       const relayUrl = relayUrls[target];
-        const runTargetDir = path.join(artifactsDir, target, `run-${runIndex}`);
+      const runTargetDir = path.join(artifactsDir, target);
       fs.mkdirSync(runTargetDir, { recursive: true });
 
       const benchEnvPrefix = [
@@ -1997,6 +1988,9 @@ async function main() {
         `PARRHESIA_BENCH_ECHO_SIZE=${opts.bench.echoSize}`,
         `PARRHESIA_BENCH_EVENT_COUNT=${opts.bench.eventCount}`,
         `PARRHESIA_BENCH_EVENT_RATE=${opts.bench.eventRate}`,
+        `PARRHESIA_BENCH_EVENT_SEND_STRATEGY=${EVENT_SEND_STRATEGY}`,
+        `PARRHESIA_BENCH_EVENT_INFLIGHT=${EVENT_INFLIGHT}`,
+        `PARRHESIA_BENCH_EVENT_ACK_TIMEOUT=${EVENT_ACK_TIMEOUT_SECONDS}`,
         `PARRHESIA_BENCH_REQ_COUNT=${opts.bench.reqCount}`,
         `PARRHESIA_BENCH_REQ_RATE=${opts.bench.reqRate}`,
         `PARRHESIA_BENCH_REQ_LIMIT=${opts.bench.reqLimit}`,
@@ -2014,7 +2008,7 @@ async function main() {
         });
 
       if (opts.quick) {
-          // Flat mode: run all benchmarks in one shot (backward compat)
+        // Flat mode: run all benchmarks in one shot
         const clientRunResults = await runSingleBenchmark({
           ...benchArgs,
           mode: "all",
@@ -2022,14 +2016,13 @@ async function main() {
         });
 
         results.push({
-            run: runIndex,
           target,
           relay_url: relayUrl,
           mode: "flat",
           clients: clientRunResults,
         });
       } else {
-          // Phased mode: separate benchmarks at different DB fill levels
+        // Phased mode: one sequence per target (seed each target DB only once)
         let eventsInDb = 0;
 
         console.log(`[bench] ${target}: connect`);
@@ -2046,29 +2039,29 @@ async function main() {
           artifactDir: path.join(runTargetDir, "echo"),
         });
 
-          // Phase: empty
+        // Phase: cold
-          console.log(`[bench] ${target}: req (empty, ${eventsInDb} events)`);
+        console.log(`[bench] ${target}: req (cold, ${eventsInDb} events)`);
-          const emptyReqResults = await runSingleBenchmark({
+        const coldReqResults = await runSingleBenchmark({
           ...benchArgs,
           mode: "req",
-            artifactDir: path.join(runTargetDir, "empty-req"),
+          artifactDir: path.join(runTargetDir, "cold-req"),
         });
 
-          console.log(`[bench] ${target}: event (empty, ${eventsInDb} events)`);
+        console.log(`[bench] ${target}: event (cold, ${eventsInDb} events)`);
-          const emptyEventResults = await runSingleBenchmark({
+        const coldEventResults = await runSingleBenchmark({
           ...benchArgs,
           mode: "event",
-            artifactDir: path.join(runTargetDir, "empty-event"),
+          artifactDir: path.join(runTargetDir, "cold-event"),
         });
-          const estimatedEmptyEventWritten = countEventsWritten(emptyEventResults);
+        const estimatedColdEventWritten = countEventsWritten(coldEventResults);
-          eventsInDb += estimatedEmptyEventWritten;
+        eventsInDb += estimatedColdEventWritten;
 
-          const authoritativeAfterEmpty = await fetchEventCountForTarget();
+        const authoritativeAfterCold = await fetchEventCountForTarget();
-          if (Number.isInteger(authoritativeAfterEmpty) && authoritativeAfterEmpty >= 0) {
+        if (Number.isInteger(authoritativeAfterCold) && authoritativeAfterCold >= 0) {
-            eventsInDb = authoritativeAfterEmpty;
+          eventsInDb = authoritativeAfterCold;
         }
 
-          console.log(`[bench] ${target}: ~${eventsInDb} events in DB after empty phase`);
+        console.log(`[bench] ${target}: ~${eventsInDb} events in DB after cold phase`);
 
         // Fill to warm
         const fillWarm = await smartFill({
@@ -2144,16 +2137,15 @@ async function main() {
         });
 
         results.push({
-            run: runIndex,
           target,
           relay_url: relayUrl,
           mode: "phased",
           phases: {
             connect: { clients: connectResults },
             echo: { clients: echoResults },
-              empty: {
+            cold: {
-                req: { clients: emptyReqResults },
+              req: { clients: coldReqResults },
-                event: { clients: emptyEventResults },
+              event: { clients: coldEventResults },
               db_events_before: 0,
             },
             warm: {
@@ -2188,7 +2180,6 @@ async function main() {
         }
       }
     }
-    }
 
     const gitTag = detectedGitTag || "untagged";
     const gitCommit = parrhesiaSource.gitCommit || detectedGitCommit || "unknown";
@@ -2224,7 +2215,6 @@ async function main() {
       machine_id: os.hostname(),
       git_tag: gitTag,
       git_commit: gitCommit,
-      runs: opts.runs,
       source: {
         kind: "cloud",
         mode: parrhesiaSource.mode,
@@ -2248,9 +2238,8 @@ async function main() {
         },
       },
       bench: {
-        runs: opts.runs,
         targets: opts.targets,
-        target_order_per_run: targetOrderPerRun,
+        target_order: targetOrder,
         mode: opts.quick ? "flat" : "phased",
         warm_events: opts.warmEvents,
         hot_events: opts.hotEvents,

scripts/cloud_bench_results.mjs

@@ -210,8 +210,8 @@ export function summarisePhasedResults(results) {
     }
 
     // Per-level req and event metrics
-    for (const level of ["empty", "warm", "hot"]) {
+    for (const level of ["cold", "warm", "hot"]) {
-      const phase = phases[level];
+      const phase = phases[level] || (level === "cold" ? phases.empty : undefined);
       if (!phase) continue;
 
       const reqClients = (phase.req?.clients || [])

scripts/just_help.sh

@@ -27,7 +27,7 @@ Examples:
   just e2e marmot
   just e2e node-sync
   just bench compare
-  just bench cloud --clients 3 --runs 3
+  just bench cloud --clients 3
 EOF
   exit 0
 fi
@@ -77,7 +77,7 @@ Examples:
   just bench collect
   just bench update --machine all
   just bench at v0.5.0
-  just bench cloud --clients 3 --runs 3
+  just bench cloud --clients 3
   just bench cloud --targets parrhesia-pg,nostream,haven --nostream-ref main
 EOF
   exit 0

scripts/run_bench_cloud.sh

@@ -18,7 +18,6 @@ Behavior:
   - Adds smoke defaults when --quick is set (unless already provided):
       --server-type cx23
       --client-type cx23
-      --runs 1
       --clients 1
       --connect-count 20
       --connect-rate 20
@@ -42,7 +41,7 @@ Everything else is passed through unchanged.
 Examples:
   just bench cloud
   just bench cloud --quick
-  just bench cloud --clients 2 --runs 1 --targets parrhesia-memory
+  just bench cloud --clients 2 --targets parrhesia-memory
   just bench cloud --image ghcr.io/owner/parrhesia:latest --threads 4
   just bench cloud --no-monitoring
   just bench cloud --yes --datacenter auto
@@ -106,7 +105,6 @@ done
 if [[ "$QUICK" == "1" ]]; then
   add_default_if_missing "--server-type" "cx23"
   add_default_if_missing "--client-type" "cx23"
-  add_default_if_missing "--runs" "1"
   add_default_if_missing "--clients" "1"
 
   add_default_if_missing "--connect-count" "20"

scripts/run_bench_update.sh

@@ -207,7 +207,7 @@ const presentBaselines = [
   ...[...discoveredBaselines].filter((srv) => !preferredBaselineOrder.includes(srv)).sort((a, b) => a.localeCompare(b)),
 ];
 
-// --- Colour palette per server: [empty, warm, hot] ---
+// --- Colour palette per server: [cold, warm, hot] ---
 const serverColours = {
   "parrhesia-pg":     ["#93c5fd", "#3b82f6", "#1e40af"],
   "parrhesia-memory": ["#86efac", "#22c55e", "#166534"],
@@ -218,12 +218,12 @@ const serverColours = {
 };
 
 const levelStyles = [
-  /* empty */ { dt: 3, pt: 6, ps: 0.7, lw: 1.5 },
+  /* cold */ { dt: 3, pt: 6, ps: 0.7, lw: 1.5 },
   /* warm */ { dt: 2, pt: 8, ps: 0.8, lw: 1.5 },
   /* hot  */ { dt: 1, pt: 7, ps: 1.0, lw: 2 },
 ];
 
-const levels = ["empty", "warm", "hot"];
+const levels = ["cold", "warm", "hot"];
 
 const shortLabel = {
   "parrhesia-pg": "pg", "parrhesia-memory": "mem",
@@ -233,19 +233,31 @@ const shortLabel = {
 
 const allServers = ["parrhesia-pg", "parrhesia-memory", ...presentBaselines];
 
-function isPhased(e) {
-  for (const srv of Object.values(e.servers || {})) {
-    if (srv.event_empty_tps !== undefined) return true;
-  }
-  return false;
-}
-
-// Build phased key: "event_tps" + "empty" → "event_empty_tps"
 function phasedKey(base, level) {
   const idx = base.lastIndexOf("_");
   return `${base.slice(0, idx)}_${level}_${base.slice(idx + 1)}`;
 }
 
+function phasedValue(d, base, level) {
+  const direct = d?.[phasedKey(base, level)];
+  if (direct !== undefined) return direct;
+
+  if (level === "cold") {
+    // Backward compatibility for historical entries written with `empty` phase names.
+    const legacy = d?.[phasedKey(base, "empty")];
+    if (legacy !== undefined) return legacy;
+  }
+
+  return undefined;
+}
+
+function isPhased(e) {
+  for (const srv of Object.values(e.servers || {})) {
+    if (phasedValue(srv, "event_tps", "cold") !== undefined) return true;
+  }
+  return false;
+}
+
 // --- Emit linetype definitions (server × level) ---
 const plotLines = [];
 for (let si = 0; si < allServers.length; si++) {
@@ -297,7 +309,7 @@ for (const panel of panels) {
     plotLines.push("");
 
   } else {
-    // Three columns per server (empty, warm, hot)
+    // Three columns per server (cold, warm, hot)
     const header = ["tag"];
     for (const srv of allServers) {
       const sl = shortLabel[srv] || srv;
@@ -311,7 +323,7 @@ for (const panel of panels) {
         const d = e.servers?.[srv];
         if (!d) { row.push("NaN", "NaN", "NaN"); continue; }
         if (phased) {
-          for (const lvl of levels) row.push(d[phasedKey(panel.base, lvl)] ?? "NaN");
+          for (const lvl of levels) row.push(phasedValue(d, panel.base, lvl) ?? "NaN");
         } else {
           row.push("NaN", d[panel.base] ?? "NaN", "NaN"); // flat → warm only
         }
@@ -320,7 +332,7 @@ for (const panel of panels) {
     }
     fs.writeFileSync(path.join(workDir, panel.file), rows.join("\n") + "\n", "utf8");
 
-    // Plot: three series per server (empty/warm/hot)
+    // Plot: three series per server (cold/warm/hot)
     const dataFile = `data_dir."/${panel.file}"`;
     plotLines.push(`set title "${panel.label}"`);
     plotLines.push(`set ylabel "${panel.ylabel}"`);
@@ -395,7 +407,7 @@ if (!pg || !mem) {
 }
 
 // Detect phased entries — use hot fill level as headline metric
-const phased = pg.event_empty_tps !== undefined;
+const phased = pg.event_cold_tps !== undefined || pg.event_empty_tps !== undefined;
 
 // For phased entries, resolve "event_tps" → "event_hot_tps" etc.
 function resolveKey(key) {