diff --git a/devenv.nix b/devenv.nix index 3968fd4..06b0708 100644 --- a/devenv.nix +++ b/devenv.nix @@ -78,6 +78,7 @@ in { with pkgs; [ just + # Mix NIFs gcc git gnumake @@ -85,6 +86,8 @@ in { automake libtool pkg-config + # for tests + openssl # Nix code formatter alejandra # i18n @@ -107,6 +110,7 @@ in { hcloud ] ++ lib.optionals pkgs.stdenv.hostPlatform.isx86_64 [ + # Nostr reference servers strfry ]; diff --git a/scripts/cloud_bench_orchestrate.mjs b/scripts/cloud_bench_orchestrate.mjs new file mode 100755 index 0000000..5ae4334 --- /dev/null +++ b/scripts/cloud_bench_orchestrate.mjs @@ -0,0 +1,1101 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const ROOT_DIR = path.resolve(__dirname, ".."); + +const DEFAULT_TARGETS = ["parrhesia-pg", "parrhesia-memory", "strfry", "nostr-rs-relay"]; + +const DEFAULTS = { + datacenter: "fsn1-dc14", + serverType: "cx23", + clientType: "cx23", + imageBase: "ubuntu-24.04", + clients: 3, + runs: 3, + targets: DEFAULT_TARGETS, + historyFile: "bench/cloud_history.jsonl", + artifactsDir: "bench/cloud_artifacts", + gitRef: "HEAD", + parrhesiaImage: null, + postgresImage: "postgres:17", + strfryImage: "ghcr.io/hoytech/strfry:latest", + nostrRsImage: "scsibug/nostr-rs-relay:latest", + keep: false, + bench: { + connectCount: 200, + connectRate: 100, + echoCount: 100, + echoRate: 50, + echoSize: 512, + eventCount: 100, + eventRate: 50, + reqCount: 100, + reqRate: 50, + reqLimit: 10, + keepaliveSeconds: 5, + }, +}; + +function usage() { + console.log(`usage: + node scripts/cloud_bench_orchestrate.mjs [options] + +Creates one server node + N client nodes on Hetzner Cloud, runs nostr-bench in +parallel from clients against selected relay targets, stores raw client logs in +bench/cloud_artifacts//, and appends metadata + pointers to +bench/cloud_history.jsonl. + +Options: + --datacenter (default: ${DEFAULTS.datacenter}) + --server-type (default: ${DEFAULTS.serverType}) + --client-type (default: ${DEFAULTS.clientType}) + --image-base (default: ${DEFAULTS.imageBase}) + --clients (default: ${DEFAULTS.clients}) + --runs (default: ${DEFAULTS.runs}) + --targets (default: ${DEFAULT_TARGETS.join(",")}) + + Source selection (choose one style): + --parrhesia-image Use remote image tag directly (e.g. ghcr.io/...) + --git-ref Build local nix docker archive from git ref (default: HEAD) + + Images for comparison targets: + --postgres-image (default: ${DEFAULTS.postgresImage}) + --strfry-image (default: ${DEFAULTS.strfryImage}) + --nostr-rs-image (default: ${DEFAULTS.nostrRsImage}) + + Benchmark knobs: + --connect-count (default: ${DEFAULTS.bench.connectCount}) + --connect-rate (default: ${DEFAULTS.bench.connectRate}) + --echo-count (default: ${DEFAULTS.bench.echoCount}) + --echo-rate (default: ${DEFAULTS.bench.echoRate}) + --echo-size (default: ${DEFAULTS.bench.echoSize}) + --event-count (default: ${DEFAULTS.bench.eventCount}) + --event-rate (default: ${DEFAULTS.bench.eventRate}) + --req-count (default: ${DEFAULTS.bench.reqCount}) + --req-rate (default: ${DEFAULTS.bench.reqRate}) + --req-limit (default: ${DEFAULTS.bench.reqLimit}) + --keepalive-seconds (default: ${DEFAULTS.bench.keepaliveSeconds}) + + Output + lifecycle: + --history-file (default: ${DEFAULTS.historyFile}) + --artifacts-dir (default: ${DEFAULTS.artifactsDir}) + --keep Keep cloud resources (no cleanup) + -h, --help + +Notes: + - Requires hcloud, ssh, scp, ssh-keygen, git. + - Requires docker locally to build portable nostr-bench binary. + - If --parrhesia-image is omitted, requires nix locally. +`); +} + +function parseArgs(argv) { + const opts = JSON.parse(JSON.stringify(DEFAULTS)); + + const intOpt = (name, value) => { + const n = Number(value); + if (!Number.isInteger(n) || n < 1) { + throw new Error(`${name} must be a positive integer, got: ${value}`); + } + return n; + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + switch (arg) { + case "-h": + case "--help": + usage(); + process.exit(0); + break; + case "--datacenter": + opts.datacenter = argv[++i]; + break; + case "--server-type": + opts.serverType = argv[++i]; + break; + case "--client-type": + opts.clientType = argv[++i]; + break; + case "--image-base": + opts.imageBase = argv[++i]; + break; + case "--clients": + opts.clients = intOpt(arg, argv[++i]); + break; + case "--runs": + opts.runs = intOpt(arg, argv[++i]); + break; + case "--targets": + opts.targets = argv[++i] + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + break; + case "--parrhesia-image": + opts.parrhesiaImage = argv[++i]; + break; + case "--git-ref": + opts.gitRef = argv[++i]; + break; + case "--postgres-image": + opts.postgresImage = argv[++i]; + break; + case "--strfry-image": + opts.strfryImage = argv[++i]; + break; + case "--nostr-rs-image": + opts.nostrRsImage = argv[++i]; + break; + case "--connect-count": + opts.bench.connectCount = intOpt(arg, argv[++i]); + break; + case "--connect-rate": + opts.bench.connectRate = intOpt(arg, argv[++i]); + break; + case "--echo-count": + opts.bench.echoCount = intOpt(arg, argv[++i]); + break; + case "--echo-rate": + opts.bench.echoRate = intOpt(arg, argv[++i]); + break; + case "--echo-size": + opts.bench.echoSize = intOpt(arg, argv[++i]); + break; + case "--event-count": + opts.bench.eventCount = intOpt(arg, argv[++i]); + break; + case "--event-rate": + opts.bench.eventRate = intOpt(arg, argv[++i]); + break; + case "--req-count": + opts.bench.reqCount = intOpt(arg, argv[++i]); + break; + case "--req-rate": + opts.bench.reqRate = intOpt(arg, argv[++i]); + break; + case "--req-limit": + opts.bench.reqLimit = intOpt(arg, argv[++i]); + break; + case "--keepalive-seconds": + opts.bench.keepaliveSeconds = intOpt(arg, argv[++i]); + break; + case "--history-file": + opts.historyFile = argv[++i]; + break; + case "--artifacts-dir": + opts.artifactsDir = argv[++i]; + break; + case "--keep": + opts.keep = true; + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (!opts.targets.length) { + throw new Error("--targets must include at least one target"); + } + + for (const t of opts.targets) { + if (!DEFAULT_TARGETS.includes(t)) { + throw new Error(`invalid target: ${t} (valid: ${DEFAULT_TARGETS.join(", ")})`); + } + } + + return opts; +} + +function shellEscape(value) { + return `'${String(value).replace(/'/g, `'"'"'`)}'`; +} + +function commandExists(cmd) { + const pathEnv = process.env.PATH || ""; + for (const dir of pathEnv.split(":")) { + if (!dir) continue; + const full = path.join(dir, cmd); + try { + fs.accessSync(full, fs.constants.X_OK); + return true; + } catch { + // ignore + } + } + return false; +} + +function runCommand(command, args = [], options = {}) { + const { cwd = ROOT_DIR, env = process.env, stdio = "pipe" } = options; + + return new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd, env, stdio }); + + let stdout = ""; + let stderr = ""; + + if (child.stdout) { + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + } + + if (child.stderr) { + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + } + + child.on("error", (error) => { + reject(error); + }); + + child.on("close", (code) => { + if (code === 0) { + resolve({ code, stdout, stderr }); + } else { + const error = new Error( + `Command failed (${code}): ${command} ${args.map((a) => shellEscape(a)).join(" ")}`, + ); + error.code = code; + error.stdout = stdout; + error.stderr = stderr; + reject(error); + } + }); + }); +} + +async function sshExec(hostIp, keyPath, remoteCommand, options = {}) { + return runCommand( + "ssh", + [ + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=8", + "-i", + keyPath, + `root@${hostIp}`, + remoteCommand, + ], + options, + ); +} + +async function scpToHost(hostIp, keyPath, localPath, remotePath) { + await runCommand("scp", [ + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + keyPath, + localPath, + `root@${hostIp}:${remotePath}`, + ]); +} + +async function waitForSsh(hostIp, keyPath, attempts = 60) { + for (let i = 1; i <= attempts; i += 1) { + try { + await sshExec(hostIp, keyPath, "echo ready >/dev/null"); + return; + } catch { + await new Promise((r) => setTimeout(r, 2000)); + } + } + throw new Error(`SSH not ready after ${attempts} attempts: ${hostIp}`); +} + +async function ensureLocalPrereqs(opts) { + const required = ["hcloud", "ssh", "scp", "ssh-keygen", "git", "docker", "file"]; + const needsParrhesia = opts.targets.includes("parrhesia-pg") || opts.targets.includes("parrhesia-memory"); + + if (needsParrhesia && !opts.parrhesiaImage) { + required.push("nix"); + } + + for (const cmd of required) { + if (!commandExists(cmd)) { + throw new Error(`Required command not found in PATH: ${cmd}`); + } + } +} + +async function buildNostrBenchBinary(tmpDir) { + const srcDir = path.join(tmpDir, "nostr-bench-src"); + console.log("[local] cloning nostr-bench source..."); + await runCommand("git", ["clone", "--depth", "1", "https://github.com/rnostr/nostr-bench.git", srcDir], { + stdio: "inherit", + }); + + let binaryPath = path.join(srcDir, "target", "x86_64-unknown-linux-musl", "release", "nostr-bench"); + let buildMode = "nix-musl-static"; + + console.log("[local] building nostr-bench (attempt static musl via nix run cargo)..."); + let staticOk = false; + + if (commandExists("nix")) { + try { + await runCommand( + "nix", + ["run", "nixpkgs#cargo", "--", "build", "--release", "--target", "x86_64-unknown-linux-musl"], + { cwd: srcDir, stdio: "inherit" }, + ); + + const fileOut = await runCommand("file", [binaryPath]); + staticOk = fileOut.stdout.includes("statically linked"); + } catch { + staticOk = false; + } + } + + if (!staticOk) { + buildMode = "docker-glibc-portable"; + binaryPath = path.join(srcDir, "target", "release", "nostr-bench"); + + console.log("[local] static build unavailable, building portable glibc binary in rust:1-bookworm..."); + + await runCommand( + "docker", + [ + "run", + "--rm", + "-v", + `${srcDir}:/src`, + "-w", + "/src", + "rust:1-bookworm", + "bash", + "-lc", + "export PATH=/usr/local/cargo/bin:$PATH; apt-get update -qq >/dev/null; apt-get install -y -qq pkg-config build-essential >/dev/null; cargo build --release", + ], + { stdio: "inherit" }, + ); + + const fileOut = await runCommand("file", [binaryPath]); + if (!(fileOut.stdout.includes("/lib64/ld-linux-x86-64.so.2") || fileOut.stdout.includes("statically linked"))) { + throw new Error(`Built nostr-bench binary does not look portable: ${fileOut.stdout.trim()}`); + } + } + + const outPath = path.join(tmpDir, "nostr-bench"); + fs.copyFileSync(binaryPath, outPath); + fs.chmodSync(outPath, 0o755); + + const fileOut = await runCommand("file", [outPath]); + console.log(`[local] nostr-bench ready (${buildMode}): ${outPath}`); + console.log(`[local] ${fileOut.stdout.trim()}`); + + return { path: outPath, buildMode }; +} + +async function buildParrhesiaArchiveIfNeeded(opts, tmpDir) { + if (opts.parrhesiaImage) { + return { + mode: "remote-image", + image: opts.parrhesiaImage, + archivePath: null, + gitRef: null, + gitCommit: null, + }; + } + + const resolved = (await runCommand("git", ["rev-parse", "--verify", opts.gitRef], { cwd: ROOT_DIR })).stdout.trim(); + + let buildDir = ROOT_DIR; + let worktreeDir = null; + + if (opts.gitRef !== "HEAD") { + worktreeDir = path.join(tmpDir, "parrhesia-worktree"); + console.log(`[local] creating temporary worktree for ${opts.gitRef}...`); + await runCommand("git", ["worktree", "add", "--detach", worktreeDir, opts.gitRef], { + cwd: ROOT_DIR, + stdio: "inherit", + }); + buildDir = worktreeDir; + } + + try { + console.log(`[local] building parrhesia docker archive via nix at ${opts.gitRef}...`); + const archivePath = ( + await runCommand("nix", ["build", ".#dockerImage", "--print-out-paths", "--no-link"], { + cwd: buildDir, + }) + ).stdout.trim(); + + if (!archivePath) { + throw new Error("nix build did not return an archive path"); + } + + return { + mode: "local-git-ref", + image: "parrhesia:latest", + archivePath, + gitRef: opts.gitRef, + gitCommit: resolved, + }; + } finally { + if (worktreeDir) { + await runCommand("git", ["worktree", "remove", "--force", worktreeDir], { + cwd: ROOT_DIR, + }).catch(() => { + // ignore + }); + } + } +} + +function makeServerScript() { + return `#!/usr/bin/env bash +set -euo pipefail + +PARRHESIA_IMAGE="\${PARRHESIA_IMAGE:-parrhesia:latest}" +POSTGRES_IMAGE="\${POSTGRES_IMAGE:-postgres:17}" +STRFRY_IMAGE="\${STRFRY_IMAGE:-ghcr.io/hoytech/strfry:latest}" +NOSTR_RS_IMAGE="\${NOSTR_RS_IMAGE:-scsibug/nostr-rs-relay:latest}" + +cleanup_containers() { + docker rm -f parrhesia pg strfry nostr-rs >/dev/null 2>&1 || true +} + +wait_http() { + local url="\$1" + local timeout="\${2:-60}" + local log_container="\${3:-}" + + for _ in \$(seq 1 "\$timeout"); do + if curl -fsS "\$url" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + if [[ -n "\$log_container" ]]; then + docker logs --tail 200 "\$log_container" >&2 || true + fi + + echo "Timed out waiting for HTTP endpoint: \$url" >&2 + return 1 +} + +wait_pg() { + local timeout="\${1:-90}" + for _ in \$(seq 1 "\$timeout"); do + if docker exec pg pg_isready -U parrhesia -d parrhesia >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + docker logs --tail 200 pg >&2 || true + echo "Timed out waiting for Postgres" >&2 + return 1 +} + +wait_port() { + local port="\$1" + local timeout="\${2:-60}" + local log_container="\${3:-}" + + for _ in \$(seq 1 "\$timeout"); do + if ss -ltn | grep -q ":\${port} "; then + return 0 + fi + sleep 1 + done + + if [[ -n "\$log_container" ]]; then + docker logs --tail 200 "\$log_container" >&2 || true + fi + + echo "Timed out waiting for port: \$port" >&2 + return 1 +} + +common_parrhesia_env=() +common_parrhesia_env+=( -e PARRHESIA_ENABLE_EXPIRATION_WORKER=0 ) +common_parrhesia_env+=( -e PARRHESIA_ENABLE_PARTITION_RETENTION_WORKER=0 ) +common_parrhesia_env+=( -e PARRHESIA_PUBLIC_MAX_CONNECTIONS=infinity ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_FRAME_BYTES=16777216 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_EVENT_BYTES=4194304 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_FILTERS_PER_REQ=1024 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_FILTER_LIMIT=100000 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_TAGS_PER_EVENT=4096 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_TAG_VALUES_PER_FILTER=4096 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_IP_MAX_EVENT_INGEST_PER_WINDOW=1000000 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_RELAY_MAX_EVENT_INGEST_PER_WINDOW=1000000 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_SUBSCRIPTIONS_PER_CONNECTION=4096 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_EVENT_FUTURE_SKEW_SECONDS=31536000 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_EVENT_INGEST_PER_WINDOW=1000000 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_AUTH_MAX_AGE_SECONDS=31536000 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_OUTBOUND_QUEUE=65536 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_OUTBOUND_DRAIN_BATCH_SIZE=4096 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_NEGENTROPY_PAYLOAD_BYTES=1048576 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_NEGENTROPY_SESSIONS_PER_CONNECTION=256 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_NEGENTROPY_TOTAL_SESSIONS=100000 ) +common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_NEGENTROPY_ITEMS_PER_SESSION=1000000 ) + +cmd="\${1:-}" +if [[ -z "\$cmd" ]]; then + echo "usage: cloud-bench-server.sh " >&2 + exit 1 +fi + +case "\$cmd" in + start-parrhesia-pg) + cleanup_containers + docker network create benchnet >/dev/null 2>&1 || true + + docker run -d --name pg --network benchnet \ + -e POSTGRES_DB=parrhesia \ + -e POSTGRES_USER=parrhesia \ + -e POSTGRES_PASSWORD=parrhesia \ + "\$POSTGRES_IMAGE" >/dev/null + + wait_pg 90 + + docker run --rm --network benchnet \ + -e DATABASE_URL=ecto://parrhesia:parrhesia@pg:5432/parrhesia \ + "\$PARRHESIA_IMAGE" \ + eval "Parrhesia.Release.migrate()" + + docker run -d --name parrhesia --network benchnet \ + -p 4413:4413 \ + -e DATABASE_URL=ecto://parrhesia:parrhesia@pg:5432/parrhesia \ + -e POOL_SIZE=20 \ + "\${common_parrhesia_env[@]}" \ + "\$PARRHESIA_IMAGE" >/dev/null + + wait_http "http://127.0.0.1:4413/health" 120 parrhesia + ;; + + start-parrhesia-memory) + cleanup_containers + + docker run -d --name parrhesia \ + -p 4413:4413 \ + -e PARRHESIA_STORAGE_BACKEND=memory \ + -e PARRHESIA_MODERATION_CACHE_ENABLED=0 \ + "\${common_parrhesia_env[@]}" \ + "\$PARRHESIA_IMAGE" >/dev/null + + wait_http "http://127.0.0.1:4413/health" 120 parrhesia + ;; + + start-strfry) + cleanup_containers + + mkdir -p /root/strfry-data/strfry + cat > /root/strfry.conf <<'EOF' +# generated by cloud bench script +db = "/data/strfry" +relay { + bind = "0.0.0.0" + port = 7777 + nofiles = 131072 +} +EOF + + docker run -d --name strfry \ + -p 7777:7777 \ + -v /root/strfry.conf:/etc/strfry.conf:ro \ + -v /root/strfry-data:/data \ + "\$STRFRY_IMAGE" \ + --config /etc/strfry.conf relay >/dev/null + + wait_port 7777 60 strfry + ;; + + start-nostr-rs-relay) + cleanup_containers + + cat > /root/nostr-rs.toml <<'EOF' +[database] +engine = "sqlite" + +[network] +ip = "0.0.0.0" +port = 8080 +EOF + + docker run -d --name nostr-rs \ + -p 8080:8080 \ + -v /root/nostr-rs.toml:/usr/src/app/config.toml:ro \ + "\$NOSTR_RS_IMAGE" >/dev/null + + wait_http "http://127.0.0.1:8080/" 60 nostr-rs + ;; + + cleanup) + cleanup_containers + ;; + + *) + echo "unknown command: \$cmd" >&2 + exit 1 + ;; +esac +`; +} + +function makeClientScript() { + return `#!/usr/bin/env bash +set -euo pipefail + +relay_url="\${1:-}" +if [[ -z "\$relay_url" ]]; then + echo "usage: cloud-bench-client.sh " >&2 + exit 1 +fi + +bench_bin="\${NOSTR_BENCH_BIN:-/usr/local/bin/nostr-bench}" + +echo "==> nostr-bench connect \${relay_url}" +"\$bench_bin" connect --json \ + -c "\${PARRHESIA_BENCH_CONNECT_COUNT:-200}" \ + -r "\${PARRHESIA_BENCH_CONNECT_RATE:-100}" \ + -k "\${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + "\${relay_url}" + +echo +echo "==> nostr-bench echo \${relay_url}" +"\$bench_bin" echo --json \ + -c "\${PARRHESIA_BENCH_ECHO_COUNT:-100}" \ + -r "\${PARRHESIA_BENCH_ECHO_RATE:-50}" \ + -k "\${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + --size "\${PARRHESIA_BENCH_ECHO_SIZE:-512}" \ + "\${relay_url}" + +echo +echo "==> nostr-bench event \${relay_url}" +"\$bench_bin" event --json \ + -c "\${PARRHESIA_BENCH_EVENT_COUNT:-100}" \ + -r "\${PARRHESIA_BENCH_EVENT_RATE:-50}" \ + -k "\${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + "\${relay_url}" + +echo +echo "==> nostr-bench req \${relay_url}" +"\$bench_bin" req --json \ + -c "\${PARRHESIA_BENCH_REQ_COUNT:-100}" \ + -r "\${PARRHESIA_BENCH_REQ_RATE:-50}" \ + -k "\${PARRHESIA_BENCH_KEEPALIVE_SECONDS:-5}" \ + --limit "\${PARRHESIA_BENCH_REQ_LIMIT:-10}" \ + "\${relay_url}" +`; +} + +function parseNostrBenchSections(output) { + const lines = output.split(/\r?\n/); + let section = null; + const parsed = {}; + + for (const lineRaw of lines) { + const line = lineRaw.trim(); + const header = line.match(/^==>\s+nostr-bench\s+(connect|echo|event|req)\s+/); + if (header) { + section = header[1]; + continue; + } + + if (!line.startsWith("{")) continue; + + try { + const json = JSON.parse(line); + if (section) { + parsed[section] = json; + } + } catch { + // ignore noisy non-json lines + } + } + + return parsed; +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + await ensureLocalPrereqs(opts); + + const timestamp = new Date().toISOString(); + const runId = `cloudbench-${timestamp.replace(/[:.]/g, "-")}-${Math.floor(Math.random() * 100000)}`; + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "parrhesia-cloud-bench-")); + const localServerScriptPath = path.join(tmpDir, "cloud-bench-server.sh"); + const localClientScriptPath = path.join(tmpDir, "cloud-bench-client.sh"); + + fs.writeFileSync(localServerScriptPath, makeServerScript(), "utf8"); + fs.writeFileSync(localClientScriptPath, makeClientScript(), "utf8"); + fs.chmodSync(localServerScriptPath, 0o755); + fs.chmodSync(localClientScriptPath, 0o755); + + const artifactsRoot = path.resolve(ROOT_DIR, opts.artifactsDir); + const artifactsDir = path.join(artifactsRoot, runId); + fs.mkdirSync(artifactsDir, { recursive: true }); + + const historyFile = path.resolve(ROOT_DIR, opts.historyFile); + fs.mkdirSync(path.dirname(historyFile), { recursive: true }); + + console.log(`[run] ${runId}`); + console.log("[phase] local preparation"); + + const nostrBench = await buildNostrBenchBinary(tmpDir); + const needsParrhesia = opts.targets.includes("parrhesia-pg") || opts.targets.includes("parrhesia-memory"); + const parrhesiaSource = needsParrhesia + ? await buildParrhesiaArchiveIfNeeded(opts, tmpDir) + : { + mode: "not-needed", + image: opts.parrhesiaImage, + archivePath: null, + gitRef: null, + gitCommit: null, + }; + + const keyName = `${runId}-ssh`; + const keyPath = path.join(tmpDir, "id_ed25519"); + const keyPubPath = `${keyPath}.pub`; + + const createdServers = []; + let sshKeyCreated = false; + + const cleanup = async () => { + if (opts.keep) { + console.log("[cleanup] --keep set, skipping cloud cleanup"); + return; + } + + if (createdServers.length > 0) { + console.log("[cleanup] deleting servers..."); + await Promise.all( + createdServers.map((name) => + runCommand("hcloud", ["server", "delete", name], { stdio: "inherit" }).catch(() => { + // ignore cleanup failures + }), + ), + ); + } + + if (sshKeyCreated) { + console.log("[cleanup] deleting ssh key..."); + await runCommand("hcloud", ["ssh-key", "delete", keyName], { stdio: "inherit" }).catch(() => { + // ignore cleanup failures + }); + } + }; + + try { + console.log("[phase] create ssh credentials"); + await runCommand("ssh-keygen", ["-t", "ed25519", "-N", "", "-f", keyPath, "-C", keyName], { + stdio: "inherit", + }); + + await runCommand("hcloud", ["ssh-key", "create", "--name", keyName, "--public-key-from-file", keyPubPath], { + stdio: "inherit", + }); + sshKeyCreated = true; + + console.log("[phase] create cloud servers in parallel"); + + const serverName = `${runId}-server`; + const clientNames = Array.from({ length: opts.clients }, (_, i) => `${runId}-client-${i + 1}`); + + const createOne = (name, role, type) => + runCommand( + "hcloud", + [ + "server", + "create", + "--name", + name, + "--type", + type, + "--datacenter", + opts.datacenter, + "--image", + opts.imageBase, + "--ssh-key", + keyName, + "--label", + `bench_run=${runId}`, + "--label", + `bench_role=${role}`, + "-o", + "json", + ], + { stdio: "pipe" }, + ).then((res) => JSON.parse(res.stdout)); + + const [serverCreate, ...clientCreates] = await Promise.all([ + createOne(serverName, "server", opts.serverType), + ...clientNames.map((name) => createOne(name, "client", opts.clientType)), + ]); + + createdServers.push(serverName, ...clientNames); + + const serverIp = serverCreate.server.public_net.ipv4.ip; + const clientInfos = clientCreates.map((c) => ({ + name: c.server.name, + id: c.server.id, + ip: c.server.public_net.ipv4.ip, + })); + + console.log("[phase] wait for SSH"); + await Promise.all([ + waitForSsh(serverIp, keyPath), + ...clientInfos.map((client) => waitForSsh(client.ip, keyPath)), + ]); + + console.log("[phase] install runtime dependencies on nodes"); + const installCmd = [ + "set -euo pipefail", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update -y >/dev/null", + "apt-get install -y docker.io curl jq >/dev/null", + "systemctl enable --now docker >/dev/null", + "docker --version", + ].join("; "); + + await Promise.all([ + sshExec(serverIp, keyPath, installCmd, { stdio: "inherit" }), + ...clientInfos.map((client) => sshExec(client.ip, keyPath, installCmd, { stdio: "inherit" })), + ]); + + console.log("[phase] upload control scripts + nostr-bench binary"); + + await scpToHost(serverIp, keyPath, localServerScriptPath, "/root/cloud-bench-server.sh"); + await sshExec(serverIp, keyPath, "chmod +x /root/cloud-bench-server.sh"); + + for (const client of clientInfos) { + await scpToHost(client.ip, keyPath, localClientScriptPath, "/root/cloud-bench-client.sh"); + await scpToHost(client.ip, keyPath, nostrBench.path, "/usr/local/bin/nostr-bench"); + await sshExec(client.ip, keyPath, "chmod +x /root/cloud-bench-client.sh /usr/local/bin/nostr-bench"); + } + + console.log("[phase] server image setup"); + + let parrhesiaImageOnServer = parrhesiaSource.image; + + if (needsParrhesia) { + if (parrhesiaSource.archivePath) { + console.log("[server] uploading parrhesia docker archive..."); + await scpToHost(serverIp, keyPath, parrhesiaSource.archivePath, "/root/parrhesia.tar.gz"); + await sshExec(serverIp, keyPath, "docker load -i /root/parrhesia.tar.gz", { stdio: "inherit" }); + parrhesiaImageOnServer = "parrhesia:latest"; + } else { + console.log(`[server] pulling parrhesia image ${parrhesiaImageOnServer}...`); + await sshExec(serverIp, keyPath, `docker pull ${shellEscape(parrhesiaImageOnServer)}`, { + stdio: "inherit", + }); + } + } + + console.log("[server] pre-pulling comparison images..."); + for (const image of [opts.postgresImage, opts.strfryImage, opts.nostrRsImage]) { + await sshExec(serverIp, keyPath, `docker pull ${shellEscape(image)}`, { stdio: "inherit" }); + } + + const serverDescribe = JSON.parse( + (await runCommand("hcloud", ["server", "describe", serverName, "-o", "json"])).stdout, + ); + const clientDescribes = await Promise.all( + clientInfos.map(async (c) => + JSON.parse((await runCommand("hcloud", ["server", "describe", c.name, "-o", "json"])).stdout), + ), + ); + + const versions = { + nostr_bench: ( + await sshExec(clientInfos[0].ip, keyPath, "/usr/local/bin/nostr-bench --version") + ).stdout.trim(), + }; + + const startCommands = { + "parrhesia-pg": "start-parrhesia-pg", + "parrhesia-memory": "start-parrhesia-memory", + strfry: "start-strfry", + "nostr-rs-relay": "start-nostr-rs-relay", + }; + + const relayUrls = { + "parrhesia-pg": `ws://${serverIp}:4413/relay`, + "parrhesia-memory": `ws://${serverIp}:4413/relay`, + strfry: `ws://${serverIp}:7777`, + "nostr-rs-relay": `ws://${serverIp}:8080`, + }; + + const results = []; + + console.log("[phase] benchmark execution"); + + for (let runIndex = 1; runIndex <= opts.runs; runIndex += 1) { + for (const target of opts.targets) { + console.log(`[bench] run ${runIndex}/${opts.runs} target=${target}`); + + const serverEnvPrefix = [ + `PARRHESIA_IMAGE=${shellEscape(parrhesiaImageOnServer || "parrhesia:latest")}`, + `POSTGRES_IMAGE=${shellEscape(opts.postgresImage)}`, + `STRFRY_IMAGE=${shellEscape(opts.strfryImage)}`, + `NOSTR_RS_IMAGE=${shellEscape(opts.nostrRsImage)}`, + ].join(" "); + + await sshExec( + serverIp, + keyPath, + `${serverEnvPrefix} /root/cloud-bench-server.sh ${shellEscape(startCommands[target])}`, + { stdio: "inherit" }, + ); + + const relayUrl = relayUrls[target]; + const runTargetDir = path.join(artifactsDir, target, `run-${runIndex}`); + fs.mkdirSync(runTargetDir, { recursive: true }); + + const benchEnvPrefix = [ + `PARRHESIA_BENCH_CONNECT_COUNT=${opts.bench.connectCount}`, + `PARRHESIA_BENCH_CONNECT_RATE=${opts.bench.connectRate}`, + `PARRHESIA_BENCH_ECHO_COUNT=${opts.bench.echoCount}`, + `PARRHESIA_BENCH_ECHO_RATE=${opts.bench.echoRate}`, + `PARRHESIA_BENCH_ECHO_SIZE=${opts.bench.echoSize}`, + `PARRHESIA_BENCH_EVENT_COUNT=${opts.bench.eventCount}`, + `PARRHESIA_BENCH_EVENT_RATE=${opts.bench.eventRate}`, + `PARRHESIA_BENCH_REQ_COUNT=${opts.bench.reqCount}`, + `PARRHESIA_BENCH_REQ_RATE=${opts.bench.reqRate}`, + `PARRHESIA_BENCH_REQ_LIMIT=${opts.bench.reqLimit}`, + `PARRHESIA_BENCH_KEEPALIVE_SECONDS=${opts.bench.keepaliveSeconds}`, + ].join(" "); + + const clientRunResults = await Promise.all( + clientInfos.map(async (client) => { + const startedAt = new Date().toISOString(); + const startMs = Date.now(); + const stdoutPath = path.join(runTargetDir, `${client.name}.stdout.log`); + const stderrPath = path.join(runTargetDir, `${client.name}.stderr.log`); + + try { + const benchRes = await sshExec( + client.ip, + keyPath, + `${benchEnvPrefix} /root/cloud-bench-client.sh ${shellEscape(relayUrl)}`, + ); + + fs.writeFileSync(stdoutPath, benchRes.stdout, "utf8"); + fs.writeFileSync(stderrPath, benchRes.stderr, "utf8"); + + return { + client_name: client.name, + client_ip: client.ip, + status: "ok", + started_at: startedAt, + finished_at: new Date().toISOString(), + duration_ms: Date.now() - startMs, + stdout_path: path.relative(ROOT_DIR, stdoutPath), + stderr_path: path.relative(ROOT_DIR, stderrPath), + sections: parseNostrBenchSections(benchRes.stdout), + }; + } catch (error) { + const out = error.stdout || ""; + const err = error.stderr || String(error); + fs.writeFileSync(stdoutPath, out, "utf8"); + fs.writeFileSync(stderrPath, err, "utf8"); + + return { + client_name: client.name, + client_ip: client.ip, + status: "error", + started_at: startedAt, + finished_at: new Date().toISOString(), + duration_ms: Date.now() - startMs, + stdout_path: path.relative(ROOT_DIR, stdoutPath), + stderr_path: path.relative(ROOT_DIR, stderrPath), + error: String(error.message || error), + sections: parseNostrBenchSections(out), + }; + } + }), + ); + + results.push({ + run: runIndex, + target, + relay_url: relayUrl, + clients: clientRunResults, + }); + + const failed = clientRunResults.filter((r) => r.status !== "ok"); + if (failed.length > 0) { + throw new Error( + `Client benchmark failed for target=${target}, run=${runIndex}: ${failed + .map((f) => f.client_name) + .join(", ")}`, + ); + } + } + } + + console.log("[phase] final server cleanup (containers)"); + await sshExec(serverIp, keyPath, "/root/cloud-bench-server.sh cleanup"); + + const entry = { + timestamp, + run_id: runId, + machine_id: os.hostname(), + source: { + mode: parrhesiaSource.mode, + parrhesia_image: parrhesiaImageOnServer, + git_ref: parrhesiaSource.gitRef, + git_commit: parrhesiaSource.gitCommit, + }, + infra: { + datacenter: opts.datacenter, + server_type: opts.serverType, + client_type: opts.clientType, + image_base: opts.imageBase, + clients: opts.clients, + }, + bench: { + runs: opts.runs, + targets: opts.targets, + ...opts.bench, + }, + versions, + artifacts_dir: path.relative(ROOT_DIR, artifactsDir), + hcloud: { + server: serverDescribe, + clients: clientDescribes, + }, + results, + }; + + fs.appendFileSync(historyFile, `${JSON.stringify(entry)}\n`, "utf8"); + + console.log("[done] benchmark complete"); + console.log(`[done] history appended: ${path.relative(ROOT_DIR, historyFile)}`); + console.log(`[done] artifacts: ${path.relative(ROOT_DIR, artifactsDir)}`); + if (opts.keep) { + console.log(`[done] resources kept. server=${serverName} clients=${clientNames.join(",")}`); + console.log(`[done] ssh key kept: ${keyName}`); + } + } finally { + await cleanup(); + } +} + +main().catch((error) => { + console.error("[error]", error?.message || error); + if (error?.stderr) { + console.error(error.stderr); + } + process.exit(1); +});