bench: Cloud firewall

scripts/cloud_bench_orchestrate.mjs

@@ -54,6 +54,7 @@ const DEFAULTS = {
   keep: false,
   quick: false,
   monitoring: true,
+  yes: false,
   warmEvents: 25000,
   hotEvents: 250000,
   bench: {
@@ -82,7 +83,7 @@ bench/cloud_artifacts/<run_id>/, and appends metadata + pointers to
 bench/history.jsonl.
 
 Options:
-  --datacenter <name>          Initial datacenter selection (default: ${DEFAULTS.datacenter})
+  --datacenter <name|auto>     Initial datacenter selection (default: ${DEFAULTS.datacenter})
   --server-type <name>         (default: ${DEFAULTS.serverType})
   --client-type <name>         (default: ${DEFAULTS.clientType})
   --image-base <name>          (default: ${DEFAULTS.imageBase})
@@ -126,15 +127,17 @@ Options:
   --artifacts-dir <path>       (default: ${DEFAULTS.artifactsDir})
   --keep                       Keep cloud resources (no cleanup)
   --no-monitoring              Skip Prometheus + node_exporter setup
+  --yes                        Skip interactive prompts and proceed immediately
   -h, --help
 
 Notes:
   - Requires hcloud, ssh, scp, ssh-keygen, git.
   - Before provisioning, checks all datacenters for type availability and estimates ${ESTIMATE_WINDOW_LABEL} cost.
-  - In interactive terminals, prompts you to pick + confirm the datacenter.
+  - In interactive terminals, prompts you to pick + confirm the datacenter unless --yes is set.
   - Caches built nostr-bench at _build/bench/nostr-bench and reuses it when valid.
   - Auto-tunes Postgres/Redis/app pool sizing from server RAM + CPU for DB-backed targets.
   - Randomizes target order per run and wipes persisted target data directories on each start.
+  - Creates a Hetzner Cloud firewall restricting inbound access to benchmark ports from known IPs only.
   - Handles Ctrl-C / SIGTERM with best-effort cloud cleanup.
   - Tries nix .#nostrBenchStaticX86_64Musl first; falls back to docker-built portable nostr-bench.
   - If --parrhesia-image is omitted, requires nix locally.
@@ -267,6 +270,9 @@ function parseArgs(argv) {
       case "--no-monitoring":
         opts.monitoring = false;
         break;
+      case "--yes":
+        opts.yes = true;
+        break;
       case "--warm-events":
         opts.warmEvents = intOpt(arg, argv[++i]);
         break;
@@ -557,10 +563,15 @@ async function chooseDatacenter(opts) {
 
   printDatacenterChoices(choices, opts);
 
-  const defaultChoice = choices.find((choice) => choice.name === opts.datacenter) || choices[0];
+  const wantsAutoDatacenter = opts.datacenter === "auto";
+  const defaultChoice = wantsAutoDatacenter
+    ? choices[0]
+    : choices.find((choice) => choice.name === opts.datacenter) || choices[0];
 
-  if (!process.stdin.isTTY || !process.stdout.isTTY) {
+  const nonInteractiveOrYes = !process.stdin.isTTY || !process.stdout.isTTY || opts.yes;
-    if (!choices.some((choice) => choice.name === opts.datacenter)) {
+
+  if (nonInteractiveOrYes) {
+    if (!wantsAutoDatacenter && !choices.some((choice) => choice.name === opts.datacenter)) {
       throw new Error(
         `Requested datacenter ${opts.datacenter} is not currently compatible. Compatible: ${choices
           .map((choice) => choice.name)
@@ -568,8 +579,15 @@ async function chooseDatacenter(opts) {
       );
     }
 
+    const modeLabel = opts.yes && process.stdin.isTTY && process.stdout.isTTY
+      ? "auto-confirm mode (--yes)"
+      : "non-interactive mode";
+    const selectionLabel = wantsAutoDatacenter
+      ? "auto cheapest compatible datacenter"
+      : `requested datacenter ${defaultChoice.name}`;
+
     console.log(
-      `[plan] non-interactive mode: using datacenter ${opts.datacenter} (${ESTIMATE_WINDOW_LABEL} est gross=${formatEuro(defaultChoice.estimatedTotal.gross)} net=${formatEuro(defaultChoice.estimatedTotal.net)})`,
+      `[plan] ${modeLabel}: using ${selectionLabel} (${ESTIMATE_WINDOW_LABEL} est gross=${formatEuro(defaultChoice.estimatedTotal.gross)} net=${formatEuro(defaultChoice.estimatedTotal.net)})`,
     );
     return defaultChoice;
   }
@@ -1396,6 +1414,8 @@ async function main() {
 
   const createdServers = [];
   let sshKeyCreated = false;
+  let firewallName = null;
+  let firewallCreated = false;
   let cleanupPromise = null;
 
   const cleanup = async () => {
@@ -1424,6 +1444,17 @@ async function main() {
         );
       }
 
+      if (firewallCreated) {
+        console.log("[cleanup] deleting firewall...");
+        await runCommand("hcloud", ["firewall", "delete", firewallName])
+          .then(() => {
+            console.log(`[cleanup] deleted firewall: ${firewallName}`);
+          })
+          .catch((error) => {
+            console.warn(`[cleanup] failed to delete firewall ${firewallName}: ${error.message || error}`);
+          });
+      }
+
       if (sshKeyCreated) {
         console.log("[cleanup] deleting ssh key...");
         await runCommand("hcloud", ["ssh-key", "delete", keyName])
@@ -1530,6 +1561,42 @@ async function main() {
       ...clientInfos.map((client) => waitForSsh(client.ip, keyPath)),
     ]);
 
+    // Detect orchestrator public IP from the server's perspective.
+    const orchestratorIp = (
+      await sshExec(serverIp, keyPath, "echo $SSH_CLIENT")
+    ).stdout.trim().split(/\s+/)[0];
+
+    // Create a firewall restricting inbound access to known benchmark IPs only.
+    firewallName = `${runId}-fw`;
+    const allBenchIps = [orchestratorIp, serverIp, ...clientInfos.map((c) => c.ip)];
+    const sourceIps = [...new Set(allBenchIps)].map((ip) => `${ip}/32`);
+
+    const firewallRules = [
+      { direction: "in", protocol: "tcp", port: "22",   source_ips: sourceIps, description: "SSH" },
+      { direction: "in", protocol: "tcp", port: "3355", source_ips: sourceIps, description: "Haven" },
+      { direction: "in", protocol: "tcp", port: "4413", source_ips: sourceIps, description: "Parrhesia" },
+      { direction: "in", protocol: "tcp", port: "7777", source_ips: sourceIps, description: "strfry" },
+      { direction: "in", protocol: "tcp", port: "8008", source_ips: sourceIps, description: "Nostream" },
+      { direction: "in", protocol: "tcp", port: "8080", source_ips: sourceIps, description: "nostr-rs-relay" },
+      { direction: "in", protocol: "tcp", port: "9090", source_ips: sourceIps, description: "Prometheus" },
+      { direction: "in", protocol: "tcp", port: "9100", source_ips: sourceIps, description: "node_exporter" },
+      { direction: "in", protocol: "icmp",              source_ips: ["0.0.0.0/0", "::/0"], description: "ICMP" },
+    ];
+
+    const rulesPath = path.join(tmpDir, "firewall-rules.json");
+    fs.writeFileSync(rulesPath, JSON.stringify(firewallRules));
+
+    await runCommand("hcloud", ["firewall", "create", "--name", firewallName, "--rules-file", rulesPath]);
+    firewallCreated = true;
+
+    for (const name of createdServers) {
+      await runCommand("hcloud", [
+        "firewall", "apply-to-resource", firewallName,
+        "--type", "server", "--server", name,
+      ]);
+    }
+    console.log(`[firewall] ${firewallName} applied (sources: ${sourceIps.join(", ")})`);
+
     console.log("[phase] install runtime dependencies on server node");
     const serverInstallCmd = [
       "set -euo pipefail",

scripts/just_help.sh

@@ -60,6 +60,10 @@ Benchmark commands
   just bench cloud [args...]    Cloud benchmark wrapper
   just bench cloud-quick        Cloud smoke profile
 
+Cloud tip:
+  just bench cloud --yes --datacenter auto
+    -> auto-pick cheapest compatible DC and skip interactive confirmation
+
 Cloud defaults:
   targets = parrhesia-pg,parrhesia-memory,strfry,nostr-rs-relay,nostream,haven
 

scripts/run_bench_cloud.sh

@@ -9,87 +9,68 @@ usage() {
 usage:
   ./scripts/run_bench_cloud.sh [options] [-- extra args for cloud_bench_orchestrate.mjs]
 
-Friendly wrapper around scripts/cloud_bench_orchestrate.mjs.
+Thin wrapper around scripts/cloud_bench_orchestrate.mjs.
 
-The orchestrator checks datacenter availability for your server/client types,
+Behavior:
-shows estimated 30m pricing, and asks for selection/confirmation in interactive terminals.
+  - Forwards args directly to the orchestrator.
+  - Adds convenience aliases:
+      --image IMAGE    -> --parrhesia-image IMAGE
+  - Adds smoke defaults when --quick is set (unless already provided):
+      --server-type cx23
+      --client-type cx23
+      --runs 1
+      --clients 1
+      --connect-count 20
+      --connect-rate 20
+      --echo-count 20
+      --echo-rate 20
+      --echo-size 512
+      --event-count 20
+      --event-rate 20
+      --req-count 20
+      --req-rate 20
+      --req-limit 10
+      --keepalive-seconds 2
 
-Defaults:
+Flags handled by this wrapper:
-  Inherited from scripts/cloud_bench_orchestrate.mjs.
+  --quick
-  This wrapper only passes explicit overrides (flags/env), plus --quick profile overrides.
+  --image IMAGE
-
-Flags:
-  --quick                     Quick smoke profile (cx23/cx23, 1 run, 1 client, lower load)
-  --clients N                 Override client count
-  --runs N                    Override run count
-  --targets CSV               Override targets
-  --datacenter NAME           Override datacenter
-  --server-type NAME          Override server type
-  --client-type NAME          Override client type
-  --image IMAGE               Use remote Parrhesia image (e.g. ghcr.io/...)
-  --git-ref REF               Build Parrhesia image from git ref (default: HEAD)
-  --nostream-repo URL         Override nostream repo (default: Cameri/nostream)
-  --nostream-ref REF          Override nostream ref (default: main)
-  --haven-image IMAGE         Override Haven image
-  --threads N                 Override nostr-bench worker threads (0 = auto)
-  --keep                      Keep cloud resources after run
   -h, --help
 
-Environment overrides (all optional):
+Everything else is passed through unchanged.
-  PARRHESIA_CLOUD_DATACENTER
-  PARRHESIA_CLOUD_SERVER_TYPE
-  PARRHESIA_CLOUD_CLIENT_TYPE
-  PARRHESIA_CLOUD_CLIENTS
-  PARRHESIA_BENCH_RUNS
-  PARRHESIA_CLOUD_TARGETS
-  PARRHESIA_CLOUD_PARRHESIA_IMAGE
-  PARRHESIA_CLOUD_GIT_REF
-  PARRHESIA_CLOUD_NOSTREAM_REPO
-  PARRHESIA_CLOUD_NOSTREAM_REF
-  PARRHESIA_CLOUD_HAVEN_IMAGE
-
-Bench knobs (forwarded):
-  PARRHESIA_BENCH_CONNECT_COUNT
-  PARRHESIA_BENCH_CONNECT_RATE
-  PARRHESIA_BENCH_ECHO_COUNT
-  PARRHESIA_BENCH_ECHO_RATE
-  PARRHESIA_BENCH_ECHO_SIZE
-  PARRHESIA_BENCH_EVENT_COUNT
-  PARRHESIA_BENCH_EVENT_RATE
-  PARRHESIA_BENCH_REQ_COUNT
-  PARRHESIA_BENCH_REQ_RATE
-  PARRHESIA_BENCH_REQ_LIMIT
-  PARRHESIA_BENCH_KEEPALIVE_SECONDS
-  PARRHESIA_BENCH_THREADS
 
 Examples:
-  # Default full cloud run
+  just bench cloud
-  ./scripts/run_bench_cloud.sh
+  just bench cloud --quick
-
+  just bench cloud --clients 2 --runs 1 --targets parrhesia-memory
-  # Quick smoke
+  just bench cloud --image ghcr.io/owner/parrhesia:latest --threads 4
-  ./scripts/run_bench_cloud.sh --quick
+  just bench cloud --no-monitoring
-
+  just bench cloud --yes --datacenter auto
-  # Use a GHCR image
-  ./scripts/run_bench_cloud.sh --image ghcr.io/owner/parrhesia:latest
 EOF
 }
 
-DATACENTER="${PARRHESIA_CLOUD_DATACENTER:-}"
+has_opt() {
-SERVER_TYPE="${PARRHESIA_CLOUD_SERVER_TYPE:-}"
+  local key="$1"
-CLIENT_TYPE="${PARRHESIA_CLOUD_CLIENT_TYPE:-}"
+  shift
-CLIENTS="${PARRHESIA_CLOUD_CLIENTS:-}"
+  local arg
-RUNS="${PARRHESIA_BENCH_RUNS:-}"
+  for arg in "$@"; do
-TARGETS="${PARRHESIA_CLOUD_TARGETS:-}"
+    if [[ "$arg" == "$key" || "$arg" == "$key="* ]]; then
-PARRHESIA_IMAGE="${PARRHESIA_CLOUD_PARRHESIA_IMAGE:-}"
+      return 0
-GIT_REF="${PARRHESIA_CLOUD_GIT_REF:-}"
+    fi
-NOSTREAM_REPO="${PARRHESIA_CLOUD_NOSTREAM_REPO:-}"
+  done
-NOSTREAM_REF="${PARRHESIA_CLOUD_NOSTREAM_REF:-}"
+  return 1
-HAVEN_IMAGE="${PARRHESIA_CLOUD_HAVEN_IMAGE:-}"
+}
-THREADS="${PARRHESIA_BENCH_THREADS:-}"
-KEEP=0
-QUICK=0
 
-EXTRA_ARGS=()
+add_default_if_missing() {
+  local key="$1"
+  local value="$2"
+  if ! has_opt "$key" "${ORCH_ARGS[@]}"; then
+    ORCH_ARGS+=("$key" "$value")
+  fi
+}
+
+ORCH_ARGS=()
+QUICK=0
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -99,156 +80,49 @@ while [[ $# -gt 0 ]]; do
       ;;
     --quick)
       QUICK=1
+      ORCH_ARGS+=("--quick")
       shift
       ;;
-    --clients)
-      CLIENTS="$2"
-      shift 2
-      ;;
-    --runs)
-      RUNS="$2"
-      shift 2
-      ;;
-    --targets)
-      TARGETS="$2"
-      shift 2
-      ;;
-    --datacenter)
-      DATACENTER="$2"
-      shift 2
-      ;;
-    --server-type)
-      SERVER_TYPE="$2"
-      shift 2
-      ;;
-    --client-type)
-      CLIENT_TYPE="$2"
-      shift 2
-      ;;
     --image)
-      PARRHESIA_IMAGE="$2"
+      if [[ $# -lt 2 ]]; then
+        echo "Missing value for --image" >&2
+        exit 1
+      fi
+      ORCH_ARGS+=("--parrhesia-image" "$2")
       shift 2
       ;;
-    --git-ref)
-      GIT_REF="$2"
-      shift 2
-      ;;
-    --nostream-repo)
-      NOSTREAM_REPO="$2"
-      shift 2
-      ;;
-    --nostream-ref)
-      NOSTREAM_REF="$2"
-      shift 2
-      ;;
-    --haven-image)
-      HAVEN_IMAGE="$2"
-      shift 2
-      ;;
-    --threads)
-      THREADS="$2"
-      shift 2
-      ;;
-    --keep)
-      KEEP=1
-      shift
-      ;;
     --)
       shift
-      EXTRA_ARGS+=("$@")
+      ORCH_ARGS+=("$@")
       break
       ;;
     *)
-      echo "Unknown argument: $1" >&2
+      ORCH_ARGS+=("$1")
-      usage
+      shift
-      exit 1
       ;;
   esac
 done
 
 if [[ "$QUICK" == "1" ]]; then
-  : "${SERVER_TYPE:=cx23}"
+  add_default_if_missing "--server-type" "cx23"
-  : "${CLIENT_TYPE:=cx23}"
+  add_default_if_missing "--client-type" "cx23"
-  : "${RUNS:=1}"
+  add_default_if_missing "--runs" "1"
-  : "${CLIENTS:=1}"
+  add_default_if_missing "--clients" "1"
 
-  : "${PARRHESIA_BENCH_CONNECT_COUNT:=20}"
+  add_default_if_missing "--connect-count" "20"
-  : "${PARRHESIA_BENCH_CONNECT_RATE:=20}"
+  add_default_if_missing "--connect-rate" "20"
-  : "${PARRHESIA_BENCH_ECHO_COUNT:=20}"
+  add_default_if_missing "--echo-count" "20"
-  : "${PARRHESIA_BENCH_ECHO_RATE:=20}"
+  add_default_if_missing "--echo-rate" "20"
-  : "${PARRHESIA_BENCH_ECHO_SIZE:=512}"
+  add_default_if_missing "--echo-size" "512"
-  : "${PARRHESIA_BENCH_EVENT_COUNT:=20}"
+  add_default_if_missing "--event-count" "20"
-  : "${PARRHESIA_BENCH_EVENT_RATE:=20}"
+  add_default_if_missing "--event-rate" "20"
-  : "${PARRHESIA_BENCH_REQ_COUNT:=20}"
+  add_default_if_missing "--req-count" "20"
-  : "${PARRHESIA_BENCH_REQ_RATE:=20}"
+  add_default_if_missing "--req-rate" "20"
-  : "${PARRHESIA_BENCH_REQ_LIMIT:=10}"
+  add_default_if_missing "--req-limit" "10"
-  : "${PARRHESIA_BENCH_KEEPALIVE_SECONDS:=2}"
+  add_default_if_missing "--keepalive-seconds" "2"
 fi
 
-CMD=(node scripts/cloud_bench_orchestrate.mjs)
+CMD=(node scripts/cloud_bench_orchestrate.mjs "${ORCH_ARGS[@]}")
-
-if [[ -n "$DATACENTER" ]]; then
-  CMD+=(--datacenter "$DATACENTER")
-fi
-if [[ -n "$SERVER_TYPE" ]]; then
-  CMD+=(--server-type "$SERVER_TYPE")
-fi
-if [[ -n "$CLIENT_TYPE" ]]; then
-  CMD+=(--client-type "$CLIENT_TYPE")
-fi
-if [[ -n "$CLIENTS" ]]; then
-  CMD+=(--clients "$CLIENTS")
-fi
-if [[ -n "$RUNS" ]]; then
-  CMD+=(--runs "$RUNS")
-fi
-if [[ -n "$TARGETS" ]]; then
-  CMD+=(--targets "$TARGETS")
-fi
-if [[ -n "$NOSTREAM_REPO" ]]; then
-  CMD+=(--nostream-repo "$NOSTREAM_REPO")
-fi
-if [[ -n "$NOSTREAM_REF" ]]; then
-  CMD+=(--nostream-ref "$NOSTREAM_REF")
-fi
-if [[ -n "$HAVEN_IMAGE" ]]; then
-  CMD+=(--haven-image "$HAVEN_IMAGE")
-fi
-if [[ -n "$THREADS" ]]; then
-  CMD+=(--threads "$THREADS")
-fi
-
-if [[ -n "$PARRHESIA_IMAGE" ]]; then
-  CMD+=(--parrhesia-image "$PARRHESIA_IMAGE")
-elif [[ -n "$GIT_REF" ]]; then
-  CMD+=(--git-ref "$GIT_REF")
-fi
-
-if [[ "$KEEP" == "1" ]]; then
-  CMD+=(--keep)
-fi
-
-# Forward bench knob envs if set
-for kv in \
-  PARRHESIA_BENCH_CONNECT_COUNT \
-  PARRHESIA_BENCH_CONNECT_RATE \
-  PARRHESIA_BENCH_ECHO_COUNT \
-  PARRHESIA_BENCH_ECHO_RATE \
-  PARRHESIA_BENCH_ECHO_SIZE \
-  PARRHESIA_BENCH_EVENT_COUNT \
-  PARRHESIA_BENCH_EVENT_RATE \
-  PARRHESIA_BENCH_REQ_COUNT \
-  PARRHESIA_BENCH_REQ_RATE \
-  PARRHESIA_BENCH_REQ_LIMIT \
-  PARRHESIA_BENCH_KEEPALIVE_SECONDS
- do
-  if [[ -n "${!kv:-}" ]]; then
-    flag="--$(echo "$kv" | tr '[:upper:]' '[:lower:]' | sed -E 's/^parrhesia_bench_//' | tr '_' '-')"
-    CMD+=("$flag" "${!kv}")
-  fi
- done
-
-CMD+=("${EXTRA_ARGS[@]}")
 
 printf 'Running cloud bench:\n  %q' "${CMD[0]}"
 for ((i=1; i<${#CMD[@]}; i++)); do