diff --git a/README.md b/README.md index 75cfca0..14d0653 100644 --- a/README.md +++ b/README.md @@ -551,7 +551,7 @@ Notes: ## Benchmark -The benchmark compares two Parrhesia profiles, one backed by PostgreSQL and one backed by the in-memory adapter, against [`strfry`](https://github.com/hoytech/strfry) and [`nostr-rs-relay`](https://sr.ht/~gheartsfield/nostr-rs-relay/) using [`nostr-bench`](https://github.com/rnostr/nostr-bench). Benchmark runs also lift Parrhesia's relay-side limits by default so the benchmark client, not server guardrails, is the main bottleneck. +The benchmark compares two Parrhesia profiles, one backed by PostgreSQL and one backed by the in-memory adapter, against [`strfry`](https://github.com/hoytech/strfry) and [`nostr-rs-relay`](https://sr.ht/~gheartsfield/nostr-rs-relay/) using [`nostr-bench`](https://github.com/rnostr/nostr-bench). The cloud benchmark target set also includes [`nostream`](https://github.com/Cameri/nostream) and [`Haven`](https://github.com/bitvora/haven). Benchmark runs also lift Parrhesia's relay-side limits by default so the benchmark client, not server guardrails, is the main bottleneck. `just bench compare` is a sequential mixed-workload benchmark, not an isolated per-endpoint microbenchmark. Each relay instance runs `connect`, then `echo`, then `event`, then `req` against the same live process, so later phases measure against state and load created by earlier phases. diff --git a/scripts/cloud_bench_orchestrate.mjs b/scripts/cloud_bench_orchestrate.mjs index 6a23e60..0cae021 100755 --- a/scripts/cloud_bench_orchestrate.mjs +++ b/scripts/cloud_bench_orchestrate.mjs @@ -10,7 +10,7 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const ROOT_DIR = path.resolve(__dirname, ".."); -const DEFAULT_TARGETS = ["parrhesia-pg", "parrhesia-memory", "strfry", "nostr-rs-relay"]; +const DEFAULT_TARGETS = ["parrhesia-pg", "parrhesia-memory", "strfry", "nostr-rs-relay", "nostream", "haven"]; const DEFAULTS = { datacenter: "fsn1-dc14", @@ -27,6 +27,9 @@ const DEFAULTS = { postgresImage: "postgres:17", strfryImage: "ghcr.io/hoytech/strfry:latest", nostrRsImage: "scsibug/nostr-rs-relay:latest", + nostreamRepo: "https://github.com/Cameri/nostream.git", + nostreamRef: "main", + havenImage: "holgerhatgarkeinenode/haven-docker:latest", keep: false, bench: { connectCount: 200, @@ -69,6 +72,9 @@ Options: --postgres-image (default: ${DEFAULTS.postgresImage}) --strfry-image (default: ${DEFAULTS.strfryImage}) --nostr-rs-image (default: ${DEFAULTS.nostrRsImage}) + --nostream-repo (default: ${DEFAULTS.nostreamRepo}) + --nostream-ref (default: ${DEFAULTS.nostreamRef}) + --haven-image (default: ${DEFAULTS.havenImage}) Benchmark knobs: --connect-count (default: ${DEFAULTS.bench.connectCount}) @@ -154,6 +160,15 @@ function parseArgs(argv) { case "--nostr-rs-image": opts.nostrRsImage = argv[++i]; break; + case "--nostream-repo": + opts.nostreamRepo = argv[++i]; + break; + case "--nostream-ref": + opts.nostreamRef = argv[++i]; + break; + case "--haven-image": + opts.havenImage = argv[++i]; + break; case "--connect-count": opts.bench.connectCount = intOpt(arg, argv[++i]); break; @@ -472,9 +487,20 @@ PARRHESIA_IMAGE="\${PARRHESIA_IMAGE:-parrhesia:latest}" POSTGRES_IMAGE="\${POSTGRES_IMAGE:-postgres:17}" STRFRY_IMAGE="\${STRFRY_IMAGE:-ghcr.io/hoytech/strfry:latest}" NOSTR_RS_IMAGE="\${NOSTR_RS_IMAGE:-scsibug/nostr-rs-relay:latest}" +NOSTREAM_REPO="\${NOSTREAM_REPO:-https://github.com/Cameri/nostream.git}" +NOSTREAM_REF="\${NOSTREAM_REF:-main}" +HAVEN_IMAGE="\${HAVEN_IMAGE:-holgerhatgarkeinenode/haven-docker:latest}" +HAVEN_RELAY_URL="\${HAVEN_RELAY_URL:-127.0.0.1:3355}" + +NOSTREAM_SECRET="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +HAVEN_OWNER_NPUB="npub1utx00neqgqln72j22kej3ux7803c2k986henvvha4thuwfkper4s7r50e8" cleanup_containers() { - docker rm -f parrhesia pg strfry nostr-rs >/dev/null 2>&1 || true + docker rm -f parrhesia pg strfry nostr-rs nostream nostream-db nostream-cache haven >/dev/null 2>&1 || true +} + +ensure_benchnet() { + docker network create benchnet >/dev/null 2>&1 || true } wait_http() { @@ -510,6 +536,32 @@ wait_pg() { return 1 } +wait_nostream_pg() { + local timeout="\${1:-90}" + for _ in \$(seq 1 "\$timeout"); do + if docker exec nostream-db pg_isready -U nostr_ts_relay -d nostr_ts_relay >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + docker logs --tail 200 nostream-db >&2 || true + echo "Timed out waiting for nostream Postgres" >&2 + return 1 +} + +wait_nostream_redis() { + local timeout="\${1:-60}" + for _ in \$(seq 1 "\$timeout"); do + if docker exec nostream-cache redis-cli -a nostr_ts_relay ping >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + docker logs --tail 200 nostream-cache >&2 || true + echo "Timed out waiting for nostream Redis" >&2 + return 1 +} + wait_port() { local port="\$1" local timeout="\${2:-60}" @@ -555,7 +607,7 @@ common_parrhesia_env+=( -e PARRHESIA_LIMITS_MAX_NEGENTROPY_ITEMS_PER_SESSION=100 cmd="\${1:-}" if [[ -z "\$cmd" ]]; then - echo "usage: cloud-bench-server.sh " >&2 + echo "usage: cloud-bench-server.sh " >&2 exit 1 fi @@ -644,6 +696,173 @@ EOF wait_http "http://127.0.0.1:8080/" 60 nostr-rs ;; + start-nostream) + cleanup_containers + ensure_benchnet + + if [[ ! -d /root/nostream-src/.git ]]; then + git clone --depth 1 "\$NOSTREAM_REPO" /root/nostream-src >/dev/null + fi + + git -C /root/nostream-src fetch --depth 1 origin "\$NOSTREAM_REF" >/dev/null 2>&1 || true + if git -C /root/nostream-src rev-parse --verify FETCH_HEAD >/dev/null 2>&1; then + git -C /root/nostream-src checkout --force FETCH_HEAD >/dev/null + else + git -C /root/nostream-src checkout --force "\$NOSTREAM_REF" >/dev/null + fi + + nostream_ref_marker=/root/nostream-src/.bench_ref + should_build_nostream=0 + if ! docker image inspect nostream:bench >/dev/null 2>&1; then + should_build_nostream=1 + elif [[ ! -f "\$nostream_ref_marker" ]] || [[ "$(cat "\$nostream_ref_marker")" != "\$NOSTREAM_REF" ]]; then + should_build_nostream=1 + fi + + if [[ "\$should_build_nostream" == "1" ]]; then + docker build -t nostream:bench /root/nostream-src >/dev/null + printf '%s\n' "\$NOSTREAM_REF" > "\$nostream_ref_marker" + fi + + mkdir -p /root/nostream-config + if [[ ! -f /root/nostream-config/settings.yaml ]]; then + cp /root/nostream-src/resources/default-settings.yaml /root/nostream-config/settings.yaml + fi + + docker run -d --name nostream-db --network benchnet \ + -e POSTGRES_DB=nostr_ts_relay \ + -e POSTGRES_USER=nostr_ts_relay \ + -e POSTGRES_PASSWORD=nostr_ts_relay \ + "\$POSTGRES_IMAGE" >/dev/null + + wait_nostream_pg 90 + + docker run -d --name nostream-cache --network benchnet \ + redis:7.0.5-alpine3.16 \ + redis-server --loglevel warning --requirepass nostr_ts_relay >/dev/null + + wait_nostream_redis 60 + + docker run --rm --network benchnet \ + -e DB_HOST=nostream-db \ + -e DB_PORT=5432 \ + -e DB_USER=nostr_ts_relay \ + -e DB_PASSWORD=nostr_ts_relay \ + -e DB_NAME=nostr_ts_relay \ + -v /root/nostream-src/migrations:/code/migrations:ro \ + -v /root/nostream-src/knexfile.js:/code/knexfile.js:ro \ + node:18-alpine3.16 \ + sh -lc 'cd /code && npm install --no-save --quiet knex@2.4.0 pg@8.8.0 && npx knex migrate:latest' + + docker run -d --name nostream --network benchnet \ + -p 8008:8008 \ + -e SECRET="\$NOSTREAM_SECRET" \ + -e RELAY_PORT=8008 \ + -e NOSTR_CONFIG_DIR=/home/node/.nostr \ + -e DB_HOST=nostream-db \ + -e DB_PORT=5432 \ + -e DB_USER=nostr_ts_relay \ + -e DB_PASSWORD=nostr_ts_relay \ + -e DB_NAME=nostr_ts_relay \ + -e DB_MIN_POOL_SIZE=16 \ + -e DB_MAX_POOL_SIZE=64 \ + -e DB_ACQUIRE_CONNECTION_TIMEOUT=60000 \ + -e REDIS_HOST=nostream-cache \ + -e REDIS_PORT=6379 \ + -e REDIS_USER=default \ + -e REDIS_PASSWORD=nostr_ts_relay \ + -v /root/nostream-config:/home/node/.nostr:ro \ + nostream:bench >/dev/null + + wait_port 8008 180 nostream + ;; + + start-haven) + cleanup_containers + + mkdir -p /root/haven-bench/db + mkdir -p /root/haven-bench/blossom + mkdir -p /root/haven-bench/templates/static + + if [[ ! -f /root/haven-bench/templates/index.html ]]; then + cat > /root/haven-bench/templates/index.html <<'EOF' + + + + + Haven + + +

Haven

+ + +EOF + fi + + printf '[]\n' > /root/haven-bench/relays_import.json + printf '[]\n' > /root/haven-bench/relays_blastr.json + printf '[]\n' > /root/haven-bench/blacklisted_npubs.json + printf '[]\n' > /root/haven-bench/whitelisted_npubs.json + + cat > /root/haven-bench/haven.env </dev/null + + wait_port 3355 120 haven + ;; + cleanup) cleanup_containers ;; @@ -947,7 +1166,7 @@ async function main() { "set -euo pipefail", "export DEBIAN_FRONTEND=noninteractive", "apt-get update -y >/dev/null", - "apt-get install -y docker.io curl jq >/dev/null", + "apt-get install -y docker.io curl jq git >/dev/null", "systemctl enable --now docker >/dev/null", "docker --version", ].join("; "); @@ -987,7 +1206,26 @@ async function main() { } console.log("[server] pre-pulling comparison images..."); - for (const image of [opts.postgresImage, opts.strfryImage, opts.nostrRsImage]) { + const comparisonImages = new Set(); + + if (opts.targets.includes("parrhesia-pg") || opts.targets.includes("nostream")) { + comparisonImages.add(opts.postgresImage); + } + if (opts.targets.includes("strfry")) { + comparisonImages.add(opts.strfryImage); + } + if (opts.targets.includes("nostr-rs-relay")) { + comparisonImages.add(opts.nostrRsImage); + } + if (opts.targets.includes("nostream")) { + comparisonImages.add("redis:7.0.5-alpine3.16"); + comparisonImages.add("node:18-alpine3.16"); + } + if (opts.targets.includes("haven")) { + comparisonImages.add(opts.havenImage); + } + + for (const image of comparisonImages) { await sshExec(serverIp, keyPath, `docker pull ${shellEscape(image)}`, { stdio: "inherit" }); } @@ -1011,6 +1249,8 @@ async function main() { "parrhesia-memory": "start-parrhesia-memory", strfry: "start-strfry", "nostr-rs-relay": "start-nostr-rs-relay", + nostream: "start-nostream", + haven: "start-haven", }; const relayUrls = { @@ -1018,6 +1258,8 @@ async function main() { "parrhesia-memory": `ws://${serverIp}:4413/relay`, strfry: `ws://${serverIp}:7777`, "nostr-rs-relay": `ws://${serverIp}:8080`, + nostream: `ws://${serverIp}:8008`, + haven: `ws://${serverIp}:3355`, }; const results = []; @@ -1033,6 +1275,10 @@ async function main() { `POSTGRES_IMAGE=${shellEscape(opts.postgresImage)}`, `STRFRY_IMAGE=${shellEscape(opts.strfryImage)}`, `NOSTR_RS_IMAGE=${shellEscape(opts.nostrRsImage)}`, + `NOSTREAM_REPO=${shellEscape(opts.nostreamRepo)}`, + `NOSTREAM_REF=${shellEscape(opts.nostreamRef)}`, + `HAVEN_IMAGE=${shellEscape(opts.havenImage)}`, + `HAVEN_RELAY_URL=${shellEscape(`${serverIp}:3355`)}`, ].join(" "); await sshExec( diff --git a/scripts/just_help.sh b/scripts/just_help.sh index 4c6cbea..331fbd6 100755 --- a/scripts/just_help.sh +++ b/scripts/just_help.sh @@ -60,6 +60,9 @@ Benchmark commands just bench cloud [args...] Cloud benchmark wrapper just bench cloud-quick Cloud smoke profile +Cloud defaults: + targets = parrhesia-pg,parrhesia-memory,strfry,nostr-rs-relay,nostream,haven + Relay-target helpers: just bench relay [all|connect|echo|event|req] [nostr-bench-args...] just bench relay-strfry [...] @@ -71,6 +74,7 @@ Examples: just bench update --machine all just bench at v0.5.0 just bench cloud --clients 3 --runs 3 + just bench cloud --targets parrhesia-pg,nostream,haven --nostream-ref main EOF exit 0 fi diff --git a/scripts/run_bench_cloud.sh b/scripts/run_bench_cloud.sh index c372bd5..ef2d135 100755 --- a/scripts/run_bench_cloud.sh +++ b/scripts/run_bench_cloud.sh @@ -16,7 +16,7 @@ Defaults (override via env or flags): server/client type: cx23 clients: 3 runs: 3 - targets: parrhesia-pg,parrhesia-memory,strfry,nostr-rs-relay + targets: parrhesia-pg,parrhesia-memory,strfry,nostr-rs-relay,nostream,haven Flags: --quick Quick smoke profile (1 run, 1 client, lower load) @@ -28,6 +28,9 @@ Flags: --client-type NAME Override client type --image IMAGE Use remote Parrhesia image (e.g. ghcr.io/...) --git-ref REF Build Parrhesia image from git ref (default: HEAD) + --nostream-repo URL Override nostream repo (default: Cameri/nostream) + --nostream-ref REF Override nostream ref (default: main) + --haven-image IMAGE Override Haven image --keep Keep cloud resources after run -h, --help @@ -37,9 +40,12 @@ Environment overrides: PARRHESIA_CLOUD_CLIENT_TYPE (default: cx23) PARRHESIA_CLOUD_CLIENTS (default: 3) PARRHESIA_BENCH_RUNS (default: 3) - PARRHESIA_CLOUD_TARGETS (default: all 4) + PARRHESIA_CLOUD_TARGETS (default: all 6) PARRHESIA_CLOUD_PARRHESIA_IMAGE (optional) PARRHESIA_CLOUD_GIT_REF (default: HEAD) + PARRHESIA_CLOUD_NOSTREAM_REPO (default: https://github.com/Cameri/nostream.git) + PARRHESIA_CLOUD_NOSTREAM_REF (default: main) + PARRHESIA_CLOUD_HAVEN_IMAGE (default: holgerhatgarkeinenode/haven-docker:latest) Bench knobs (forwarded): PARRHESIA_BENCH_CONNECT_COUNT @@ -71,9 +77,12 @@ SERVER_TYPE="${PARRHESIA_CLOUD_SERVER_TYPE:-cx23}" CLIENT_TYPE="${PARRHESIA_CLOUD_CLIENT_TYPE:-cx23}" CLIENTS="${PARRHESIA_CLOUD_CLIENTS:-3}" RUNS="${PARRHESIA_BENCH_RUNS:-3}" -TARGETS="${PARRHESIA_CLOUD_TARGETS:-parrhesia-pg,parrhesia-memory,strfry,nostr-rs-relay}" +TARGETS="${PARRHESIA_CLOUD_TARGETS:-parrhesia-pg,parrhesia-memory,strfry,nostr-rs-relay,nostream,haven}" PARRHESIA_IMAGE="${PARRHESIA_CLOUD_PARRHESIA_IMAGE:-}" GIT_REF="${PARRHESIA_CLOUD_GIT_REF:-HEAD}" +NOSTREAM_REPO="${PARRHESIA_CLOUD_NOSTREAM_REPO:-https://github.com/Cameri/nostream.git}" +NOSTREAM_REF="${PARRHESIA_CLOUD_NOSTREAM_REF:-main}" +HAVEN_IMAGE="${PARRHESIA_CLOUD_HAVEN_IMAGE:-holgerhatgarkeinenode/haven-docker:latest}" KEEP=0 QUICK=0 @@ -121,6 +130,18 @@ while [[ $# -gt 0 ]]; do GIT_REF="$2" shift 2 ;; + --nostream-repo) + NOSTREAM_REPO="$2" + shift 2 + ;; + --nostream-ref) + NOSTREAM_REF="$2" + shift 2 + ;; + --haven-image) + HAVEN_IMAGE="$2" + shift 2 + ;; --keep) KEEP=1 shift @@ -162,6 +183,9 @@ CMD=( --clients "$CLIENTS" --runs "$RUNS" --targets "$TARGETS" + --nostream-repo "$NOSTREAM_REPO" + --nostream-ref "$NOSTREAM_REF" + --haven-image "$HAVEN_IMAGE" ) if [[ -n "$PARRHESIA_IMAGE" ]]; then