Add memory-backed benchmark profile
This commit is contained in:
@@ -10,9 +10,10 @@ usage:
|
||||
./scripts/run_bench_compare.sh
|
||||
|
||||
Runs the same nostr-bench suite against:
|
||||
1) Parrhesia (temporary prod relay via run_e2e_suite.sh)
|
||||
2) strfry (ephemeral instance) — optional, skipped if not in PATH
|
||||
3) nostr-rs-relay (ephemeral sqlite instance) — optional, skipped if not in PATH
|
||||
1) Parrhesia (Postgres, temporary prod relay via run_e2e_suite.sh)
|
||||
2) Parrhesia (in-memory storage, temporary prod relay via run_e2e_suite.sh)
|
||||
3) strfry (ephemeral instance) — optional, skipped if not in PATH
|
||||
4) nostr-rs-relay (ephemeral sqlite instance) — optional, skipped if not in PATH
|
||||
|
||||
Environment:
|
||||
PARRHESIA_BENCH_RUNS Number of comparison runs (default: 2)
|
||||
@@ -247,7 +248,7 @@ echo " ${NOSTR_BENCH_VERSION}"
|
||||
echo
|
||||
|
||||
for run in $(seq 1 "$RUNS"); do
|
||||
echo "[run ${run}/${RUNS}] Parrhesia"
|
||||
echo "[run ${run}/${RUNS}] Parrhesia (Postgres)"
|
||||
parrhesia_log="$WORK_DIR/parrhesia_${run}.log"
|
||||
if ! ./scripts/run_nostr_bench.sh all >"$parrhesia_log" 2>&1; then
|
||||
echo "Parrhesia benchmark failed. Log: $parrhesia_log" >&2
|
||||
@@ -255,6 +256,14 @@ for run in $(seq 1 "$RUNS"); do
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[run ${run}/${RUNS}] Parrhesia (Memory)"
|
||||
parrhesia_memory_log="$WORK_DIR/parrhesia_memory_${run}.log"
|
||||
if ! PARRHESIA_BENCH_STORAGE_BACKEND=memory ./scripts/run_nostr_bench.sh all >"$parrhesia_memory_log" 2>&1; then
|
||||
echo "Parrhesia memory benchmark failed. Log: $parrhesia_memory_log" >&2
|
||||
tail -n 120 "$parrhesia_memory_log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( HAS_STRFRY )); then
|
||||
echo "[run ${run}/${RUNS}] strfry"
|
||||
strfry_log="$WORK_DIR/strfry_${run}.log"
|
||||
@@ -364,6 +373,7 @@ function loadRuns(prefix) {
|
||||
}
|
||||
|
||||
const parrhesiaRuns = loadRuns("parrhesia");
|
||||
const parrhesiaMemoryRuns = loadRuns("parrhesia_memory");
|
||||
const strfryRuns = hasStrfry ? loadRuns("strfry") : [];
|
||||
const nostrRsRuns = hasNostrRs ? loadRuns("nostr_rs_relay") : [];
|
||||
|
||||
@@ -382,7 +392,10 @@ function summarise(allRuns) {
|
||||
return out;
|
||||
}
|
||||
|
||||
const summary = { parrhesia: summarise(parrhesiaRuns) };
|
||||
const summary = {
|
||||
parrhesia: summarise(parrhesiaRuns),
|
||||
parrhesiaMemory: summarise(parrhesiaMemoryRuns),
|
||||
};
|
||||
if (hasStrfry) summary.strfry = summarise(strfryRuns);
|
||||
if (hasNostrRs) summary.nostrRsRelay = summarise(nostrRsRuns);
|
||||
|
||||
@@ -404,16 +417,22 @@ const metricLabels = [
|
||||
["req throughput (MiB/s) ↑", "reqSizeMiBS"],
|
||||
];
|
||||
|
||||
const headers = ["metric", "parrhesia"];
|
||||
const headers = ["metric", "parrhesia-pg", "parrhesia-memory"];
|
||||
if (hasStrfry) headers.push("strfry");
|
||||
if (hasNostrRs) headers.push("nostr-rs-relay");
|
||||
headers.push("memory/parrhesia");
|
||||
if (hasStrfry) headers.push("strfry/parrhesia");
|
||||
if (hasNostrRs) headers.push("nostr-rs/parrhesia");
|
||||
|
||||
const rows = metricLabels.map(([label, key]) => {
|
||||
const row = [label, toFixed(summary.parrhesia[key])];
|
||||
const row = [
|
||||
label,
|
||||
toFixed(summary.parrhesia[key]),
|
||||
toFixed(summary.parrhesiaMemory[key]),
|
||||
];
|
||||
if (hasStrfry) row.push(toFixed(summary.strfry[key]));
|
||||
if (hasNostrRs) row.push(toFixed(summary.nostrRsRelay[key]));
|
||||
row.push(ratioVsParrhesia("parrhesiaMemory", key));
|
||||
if (hasStrfry) row.push(ratioVsParrhesia("strfry", key));
|
||||
if (hasNostrRs) row.push(ratioVsParrhesia("nostrRsRelay", key));
|
||||
return row;
|
||||
@@ -444,8 +463,10 @@ if (hasStrfry || hasNostrRs) {
|
||||
console.log("Run details:");
|
||||
for (let i = 0; i < runs; i += 1) {
|
||||
const p = parrhesiaRuns[i];
|
||||
const pm = parrhesiaMemoryRuns[i];
|
||||
let line = ` run ${i + 1}: ` +
|
||||
`parrhesia(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)})`;
|
||||
`parrhesia-pg(echo_tps=${toFixed(p.echoTps, 0)}, event_tps=${toFixed(p.eventTps, 0)}, req_tps=${toFixed(p.reqTps, 0)}, connect_avg_ms=${toFixed(p.connectAvgMs, 0)})` +
|
||||
` | parrhesia-memory(echo_tps=${toFixed(pm.echoTps, 0)}, event_tps=${toFixed(pm.eventTps, 0)}, req_tps=${toFixed(pm.reqTps, 0)}, connect_avg_ms=${toFixed(pm.connectAvgMs, 0)})`;
|
||||
if (hasStrfry) {
|
||||
const s = strfryRuns[i];
|
||||
line += ` | strfry(echo_tps=${toFixed(s.echoTps, 0)}, event_tps=${toFixed(s.eventTps, 0)}, req_tps=${toFixed(s.reqTps, 0)}, connect_avg_ms=${toFixed(s.connectAvgMs, 0)})`;
|
||||
|
||||
@@ -19,6 +19,8 @@ if [[ "$MIX_ENV" != "test" && "$MIX_ENV" != "prod" ]]; then
|
||||
fi
|
||||
export MIX_ENV
|
||||
|
||||
SKIP_ECTO="${PARRHESIA_E2E_SKIP_ECTO:-0}"
|
||||
|
||||
SUITE_SLUG="$(printf '%s' "$SUITE_NAME" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9' '_')"
|
||||
SUITE_UPPER="$(printf '%s' "$SUITE_SLUG" | tr '[:lower:]' '[:upper:]')"
|
||||
PORT_ENV_VAR="PARRHESIA_${SUITE_UPPER}_E2E_RELAY_PORT"
|
||||
@@ -56,14 +58,16 @@ if [[ -z "${DATABASE_URL:-}" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$MIX_ENV" == "test" ]]; then
|
||||
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet --force || true
|
||||
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.create --quiet
|
||||
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.migrate --quiet
|
||||
else
|
||||
mix ecto.drop --quiet --force || true
|
||||
mix ecto.create --quiet
|
||||
mix ecto.migrate --quiet
|
||||
if [[ "$SKIP_ECTO" != "1" ]]; then
|
||||
if [[ "$MIX_ENV" == "test" ]]; then
|
||||
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet --force || true
|
||||
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.create --quiet
|
||||
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.migrate --quiet
|
||||
else
|
||||
mix ecto.drop --quiet --force || true
|
||||
mix ecto.create --quiet
|
||||
mix ecto.migrate --quiet
|
||||
fi
|
||||
fi
|
||||
|
||||
SERVER_LOG="${ROOT_DIR}/.${SUITE_SLUG}-e2e-server.log"
|
||||
@@ -75,7 +79,7 @@ cleanup() {
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [[ "${PARRHESIA_E2E_DROP_DB_ON_EXIT:-0}" == "1" ]]; then
|
||||
if [[ "$SKIP_ECTO" != "1" && "${PARRHESIA_E2E_DROP_DB_ON_EXIT:-0}" == "1" ]]; then
|
||||
if [[ "$MIX_ENV" == "test" ]]; then
|
||||
PARRHESIA_TEST_HTTP_PORT=0 mix ecto.drop --quiet --force || true
|
||||
else
|
||||
|
||||
@@ -13,6 +13,9 @@ usage:
|
||||
Runs nostr-bench against a temporary Parrhesia prod server started via
|
||||
./scripts/run_e2e_suite.sh.
|
||||
|
||||
Benchmark target:
|
||||
PARRHESIA_BENCH_STORAGE_BACKEND postgres|memory (default: postgres)
|
||||
|
||||
Pool tuning:
|
||||
POOL_SIZE optional override for prod pool size
|
||||
DB_QUEUE_TARGET_MS optional Repo queue target override
|
||||
@@ -39,6 +42,10 @@ Default "all" run can be tuned via env vars:
|
||||
PARRHESIA_BENCH_REQ_RATE (default: 50)
|
||||
PARRHESIA_BENCH_REQ_LIMIT (default: 10)
|
||||
PARRHESIA_BENCH_KEEPALIVE_SECONDS (default: 5)
|
||||
|
||||
By default benchmark runs also lift relay limits so the benchmark client, not
|
||||
relay-side ceilings, is the bottleneck. Set `PARRHESIA_BENCH_LIFT_LIMITS=0` to
|
||||
disable that behavior.
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -63,11 +70,54 @@ if [[ "$MODE" == "all" && $# -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${PGDATABASE:-}" ]]; then
|
||||
export PGDATABASE="parrhesia_bench_prod_$(date +%s)_$RANDOM"
|
||||
BENCH_STORAGE_BACKEND="${PARRHESIA_BENCH_STORAGE_BACKEND:-postgres}"
|
||||
case "$BENCH_STORAGE_BACKEND" in
|
||||
postgres|memory)
|
||||
;;
|
||||
*)
|
||||
echo "PARRHESIA_BENCH_STORAGE_BACKEND must be postgres or memory, got: ${BENCH_STORAGE_BACKEND}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
export PARRHESIA_STORAGE_BACKEND="$BENCH_STORAGE_BACKEND"
|
||||
export PARRHESIA_ENABLE_EXPIRATION_WORKER="${PARRHESIA_ENABLE_EXPIRATION_WORKER:-0}"
|
||||
export PARRHESIA_ENABLE_PARTITION_RETENTION_WORKER="${PARRHESIA_ENABLE_PARTITION_RETENTION_WORKER:-0}"
|
||||
|
||||
if [[ "${PARRHESIA_BENCH_LIFT_LIMITS:-1}" == "1" ]]; then
|
||||
export PARRHESIA_PUBLIC_MAX_CONNECTIONS="${PARRHESIA_PUBLIC_MAX_CONNECTIONS:-infinity}"
|
||||
export PARRHESIA_LIMITS_MAX_FRAME_BYTES="${PARRHESIA_LIMITS_MAX_FRAME_BYTES:-16777216}"
|
||||
export PARRHESIA_LIMITS_MAX_EVENT_BYTES="${PARRHESIA_LIMITS_MAX_EVENT_BYTES:-4194304}"
|
||||
export PARRHESIA_LIMITS_MAX_FILTERS_PER_REQ="${PARRHESIA_LIMITS_MAX_FILTERS_PER_REQ:-1024}"
|
||||
export PARRHESIA_LIMITS_MAX_FILTER_LIMIT="${PARRHESIA_LIMITS_MAX_FILTER_LIMIT:-100000}"
|
||||
export PARRHESIA_LIMITS_MAX_TAGS_PER_EVENT="${PARRHESIA_LIMITS_MAX_TAGS_PER_EVENT:-4096}"
|
||||
export PARRHESIA_LIMITS_MAX_TAG_VALUES_PER_FILTER="${PARRHESIA_LIMITS_MAX_TAG_VALUES_PER_FILTER:-4096}"
|
||||
export PARRHESIA_LIMITS_IP_MAX_EVENT_INGEST_PER_WINDOW="${PARRHESIA_LIMITS_IP_MAX_EVENT_INGEST_PER_WINDOW:-1000000}"
|
||||
export PARRHESIA_LIMITS_RELAY_MAX_EVENT_INGEST_PER_WINDOW="${PARRHESIA_LIMITS_RELAY_MAX_EVENT_INGEST_PER_WINDOW:-1000000}"
|
||||
export PARRHESIA_LIMITS_MAX_SUBSCRIPTIONS_PER_CONNECTION="${PARRHESIA_LIMITS_MAX_SUBSCRIPTIONS_PER_CONNECTION:-4096}"
|
||||
export PARRHESIA_LIMITS_MAX_EVENT_FUTURE_SKEW_SECONDS="${PARRHESIA_LIMITS_MAX_EVENT_FUTURE_SKEW_SECONDS:-31536000}"
|
||||
export PARRHESIA_LIMITS_MAX_EVENT_INGEST_PER_WINDOW="${PARRHESIA_LIMITS_MAX_EVENT_INGEST_PER_WINDOW:-1000000}"
|
||||
export PARRHESIA_LIMITS_AUTH_MAX_AGE_SECONDS="${PARRHESIA_LIMITS_AUTH_MAX_AGE_SECONDS:-31536000}"
|
||||
export PARRHESIA_LIMITS_MAX_OUTBOUND_QUEUE="${PARRHESIA_LIMITS_MAX_OUTBOUND_QUEUE:-65536}"
|
||||
export PARRHESIA_LIMITS_OUTBOUND_DRAIN_BATCH_SIZE="${PARRHESIA_LIMITS_OUTBOUND_DRAIN_BATCH_SIZE:-4096}"
|
||||
export PARRHESIA_LIMITS_MAX_NEGENTROPY_PAYLOAD_BYTES="${PARRHESIA_LIMITS_MAX_NEGENTROPY_PAYLOAD_BYTES:-1048576}"
|
||||
export PARRHESIA_LIMITS_MAX_NEGENTROPY_SESSIONS_PER_CONNECTION="${PARRHESIA_LIMITS_MAX_NEGENTROPY_SESSIONS_PER_CONNECTION:-256}"
|
||||
export PARRHESIA_LIMITS_MAX_NEGENTROPY_TOTAL_SESSIONS="${PARRHESIA_LIMITS_MAX_NEGENTROPY_TOTAL_SESSIONS:-100000}"
|
||||
export PARRHESIA_LIMITS_MAX_NEGENTROPY_ITEMS_PER_SESSION="${PARRHESIA_LIMITS_MAX_NEGENTROPY_ITEMS_PER_SESSION:-1000000}"
|
||||
fi
|
||||
|
||||
export PARRHESIA_E2E_DROP_DB_ON_EXIT="${PARRHESIA_E2E_DROP_DB_ON_EXIT:-1}"
|
||||
if [[ "$BENCH_STORAGE_BACKEND" == "memory" ]]; then
|
||||
export PARRHESIA_E2E_SKIP_ECTO="${PARRHESIA_E2E_SKIP_ECTO:-1}"
|
||||
export PARRHESIA_E2E_DROP_DB_ON_EXIT=0
|
||||
export PARRHESIA_MODERATION_CACHE_ENABLED="${PARRHESIA_MODERATION_CACHE_ENABLED:-0}"
|
||||
else
|
||||
if [[ -z "${PGDATABASE:-}" ]]; then
|
||||
export PGDATABASE="parrhesia_bench_prod_$(date +%s)_$RANDOM"
|
||||
fi
|
||||
|
||||
export PARRHESIA_E2E_SKIP_ECTO="${PARRHESIA_E2E_SKIP_ECTO:-0}"
|
||||
export PARRHESIA_E2E_DROP_DB_ON_EXIT="${PARRHESIA_E2E_DROP_DB_ON_EXIT:-1}"
|
||||
fi
|
||||
|
||||
PARRHESIA_E2E_MIX_ENV="prod" \
|
||||
exec ./scripts/run_e2e_suite.sh \
|
||||
|
||||
Reference in New Issue
Block a user