From 924e6bc337fc5997bfbabe8abe53fd42f7cc4bed Mon Sep 17 00:00:00 2001 From: Agent Zuse Date: Sat, 27 Jun 2026 23:20:52 +0200 Subject: [PATCH] feat: add generated supertest cli Add the scope/action CLI for scenarios, topologies, blocks, and groups with generated registry listings. Replace legacy scenario and alias npm scripts with the packaged supertest bin. Include removal of the obsolete rollout progress document. --- .pre-commit-config.yaml | 2 +- README.md | 102 +----- docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md | 190 ----------- package-lock.json | 3 + package.json | 25 +- src/architecture.ts | 6 + src/cli.ts | 410 +++++++++++++++++++++++ src/config.ts | 30 -- src/index.ts | 178 +--------- src/registries.ts | 3 +- src/scenarios.ts | 70 ++-- tests/cli.test.ts | 49 +++ tests/index.test.ts | 2 +- 13 files changed, 530 insertions(+), 540 deletions(-) delete mode 100644 docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md create mode 100755 src/cli.ts create mode 100644 tests/cli.test.ts diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5564ea6..36cddc6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: npm-format-check name: npm format check - entry: npm run format:check + entry: npx prettier --check . language: system pass_filenames: false files: '\.(cjs|js|json|md|mjs|ts|tsx|yaml|yml)$' diff --git a/README.md b/README.md index d7e4949..5aa0d6d 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ npm run build Check the Guix substitute servers before spending cloud time: ```bash -npm run preflight:substitutes -npm run preflight:substitutes -- --plugin sender +node --import tsx scripts/check-substitutes.ts +node --import tsx scripts/check-substitutes.ts --plugin sender ``` Delete leftover cloud resources without using Legion state: @@ -76,105 +76,39 @@ scripts/cleanup-cloud-resources --dry-run ## Basic Usage -List scenarios: +Build the CLI, then list registered scenarios, topologies, and blocks: ```bash -npm run scenario:list +npm run build +./dist/cli.js scenario list +./dist/cli.js topology list +./dist/cli.js block list ``` -Run the single-node scenario: +Run a scenario or a topology directly: ```bash -npm run scenario:single-node-init -``` - -Run the manual single-node scenario against an existing host: - -```bash -SUPERTEST_MANUAL_HOST_IP=203.0.113.10 \ - SUPERTEST_MANUAL_USERNAME=ubuntu \ - SUPERTEST_MANUAL_PASSWORD=secret \ - npm run scenario:manual-node-init -``` - -Run the single-node plugin rollout/rollback scenario: - -```bash -npm run scenario:single-node-plugin-rollout-rollback -``` - -Run the single-node Sender ingest/HLS scenario: - -```bash -npm run scenario:single-node-sender -``` - -Run the clustered Sender fanout/reboot scenario: - -```bash -npm run scenario:cluster-sender-fanout-reboot -``` - -Run the cluster lifecycle scenario: - -```bash -npm run scenario:cluster-lifecycle -``` - -Run the clustered plugin sync split-brain scenario: - -```bash -npm run scenario:cluster-plugin-rollout-sync-split-brain +./dist/cli.js scenario run single-node-init +./dist/cli.js topology run cluster-lifecycle ``` Keep created nodes around for inspection: ```bash -SUPERTEST_KEEP_NODES=1 npm run scenario:cluster-lifecycle +./dist/cli.js scenario run cluster-lifecycle --keep-nodes ``` -Run one scenario directly with the generic entrypoint: +Groups run several scenarios sequentially and fail fast on the first failing +scenario. All scenarios in a group share one `SUPERTEST_RUN_ID`, so their +artifacts land in sibling directories under the same run. ```bash -npm run scenario -- single-node-init -npm run scenario -- manual-node-init -npm run scenario -- single-node-plugin-rollout-rollback -npm run scenario -- single-node-sender -npm run scenario -- cluster-sender-fanout-reboot -npm run scenario -- cluster-plugin-rollout-sync-split-brain -npm run scenario -- cluster-lifecycle +./dist/cli.js group run tribes +./dist/cli.js group run all ``` -## Alias Groups - -Aliases run several scenarios sequentially and fail fast on the first failing -scenario. All scenarios in an alias share one `SUPERTEST_RUN_ID`, so their -artifacts land in sibling directories under the same run. Each group is the -minimal set of scenarios that preserves its coverage. - -```bash -npm run alias:tribes # Tribes core: no-plugin + supertest scenarios -npm run alias:sender # Sender ingest/HLS scenarios -npm run alias:kobold # Trust-backed Kobold dataset scenarios -npm run alias:all # deduplicated union of every named group -``` - -- `tribes` — Tribes core: `single-node-init`, - `cluster-plugin-integrated-rollout`, `cluster-plugin-rollout-sync-split-brain`, - `cluster-lifecycle`. `single-node-plugin-rollout-rollback` is omitted because - its coverage is a subset of the cluster rollout scenarios (its rollback - executor-status check is folded into `cluster-plugin-rollout-sync-split-brain`); - run it on its own with `npm run scenario:single-node-plugin-rollout-rollback`. -- `sender` — `single-node-sender`, `cluster-sender-fanout-reboot` (both kept; the - single-node run uniquely covers external endpoint types, the direct Vinyl HLS - metric, audio ingest, and a Hetzner-init origin). -- `kobold` — `cluster-kobold-public-private`. -- `all` — the deduplicated union of the named groups. This excludes - `manual-node-init` (needs a manually supplied host, `SUPERTEST_MANUAL_*`) and - the redundant `single-node-plugin-rollout-rollback`. - -Equivalent to `npm run scenario -- group `. See `npm run scenario` -(no arguments) for the current grouping. +See `./dist/cli.js help` for generated listings and the current group +membership. ## Dev Branch Helper diff --git a/docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md b/docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md deleted file mode 100644 index 0a7aee8..0000000 --- a/docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md +++ /dev/null @@ -1,190 +0,0 @@ -# Plugin Rollout/Rollback Cross-Repo Progress - -**Status:** single-node `supertest` plugin rollout/rollback is green on signed `guix-tribes` `master`; next target is clustered plugin sync validation. -**Last updated:** 2026-05-03 -**Scope repos:** `tribes-supertest`, `tribes-plugin-supertest`, `legion_kk`, `tribes`, `guix-tribes` - ---- - -## 1) Current Baseline - -### Green live validation - -`single-node-plugin-rollout-rollback` has passed on fresh infra through the public admin API using the real pinned `supertest` plugin: - -- run: `2026-05-01t133844554z-single-node-plugin-rollout-rollback` -- node: `st-20260501-820549-hx-a` -- consumed `guix-tribes` commit: `5a348e7c5427b99c84755aa12c30c37c2de7a4ca` -- baseline selected/running: `/gnu/store/39hsfwaf39h5h91gwd80ildrsjsdr85b-system` -- plugin selected/running: `/gnu/store/w51f5b2a5ynrpfky6qyawbnjhmgx0mxs-system` -- rollback selected/running: `/gnu/store/39hsfwaf39h5h91gwd80ildrsjsdr85b-system` -- validation: plugin API health/schema/write/read passed -- rollback validation: `supertest_events` and `supertest_cases` were removed by down migrations -- cleanup: Hetzner server destroyed; final node list `[]` - -### Current `guix-tribes` master baseline - -`guix-tribes` `master` has moved beyond the first green run and now includes the follow-up fixes and plugin substitute baseline: - -- `39b1ed8 fix: skip no-op pulls and stabilize generation diagnostics` -- `8849107 fix: resolve herd for rollback migrations` -- `e13c136 test: harden local-control worker state` -- `2950278 chore: Bump tribes` -- `05c493b test: avoid running guile suites on import` -- `0b4d3a7 fix: compile bundled tribes_ui plugin` -- `fa4753a feat: build channel plugins in substitute baseline` - -`fa4753a` is the current intended `guix-tribes` master baseline for the next test runs. The substitute manifest now covers the channel plugin registry packages, including `tribes-plugin-supertest`, `tribes-plugin-aether`, and `tribes-plugin-sender` plus `ffmpeg` for sender. - -### Current plugin fixture - -`tribes-plugin-supertest` commit `e042f3265db7a40d4d558132800238c6d466e8dd` provides: - -- `Supertest.Case` and `Supertest.Event` -- reversible migrations for `supertest_cases` and `supertest_events` -- `AshNostrSync` registration for both resources -- a local-only JSON API under `/plugins-api/supertest` - -The API is enough for black-box clustered checks because the runner can call it over SSH on each node's `127.0.0.1:4000`. - ---- - -## 2) Locked Decisions - -- Scenario assertions use the normal operator/automation surface: `POST /api/admin/management` with NIP-98 auth. -- `tribes-deploy-exec` remains diagnostic-only for these scenarios. -- `supertest` is the lean migration-bearing rollout fixture; `aether` and `sender` remain real packaged plugins for broader package/substitute coverage. -- Selected generation is `/var/guix/profiles/system`; running generation is `/run/current-system`. -- Guix-sensitive work must run through the pulled/current Guix profile/module universe. -- Direct explicit generation rollback must stay async/restart-tolerant and must not replay the current rollout plan. -- `tribes-local-control` service-definition updates should be treated as restart/reboot-required work, not restarted in-band while local-control is serving the switch/rollback request. -- Plugin disable is lower priority than clustered install/sync/rollback validation. - ---- - -## 3) What Works Now - -### `tribes-supertest` - -- Public-admin-API rollout flow for `single-node-plugin-rollout-rollback`. -- Rollout assertions for preview/start/status, selected/running convergence, plugin API health/schema/write/read, rollback convergence, table removal, and provider cleanup. -- Strong diagnostics around local-control readiness, socket/listener readiness, migrations, Shepherd logs, certificates, system generation comparison, Guix build logs, and daemon logs. - -### `guix-tribes` - -- External plugin pinning and registry-backed plugin packaging. -- Target-node plugin builds with the lean Mix/OTP baseline. -- Runtime plugin loading in release mode. -- `tribes-migrations` wrapper logging with real output/exit status. -- Current-Guix delegation for service upgrade, generation comparison, config evaluation, and derivation realization. -- Rollback to installed baseline profile generations. -- Plugin rollback down migrations for direct rollback from a plugin generation to a no-plugin baseline. -- Target-generation plugin state restore before rollback. -- No-op `guix pull` skip when rollout plans explicitly resolve no channel delta. -- Generation diagnostic JSON stability. -- Channel plugin substitute manifest coverage. - -### `tribes` - -- Public admin-management rollout methods are the scenario contract. -- Startup readiness returns `503` while `Tribes.Repo` is unavailable. -- Explicit rollback generation is async and does not replay the current rollout plan. -- AshNostrSync publishes and consumes host and plugin resources through registered namespaces. - -### `legion_kk` - -- Generic admin management calls are available through Legion-owned CLI/service/API wrappers. -- Provider cleanup is materially more stable across Hetzner, Scaleway, and OVH. -- Installed-system Guix checkout-cache transplant is in place. -- Admin API retry behavior handles explicit `503` readiness responses. - ---- - -## 4) Remaining Gaps Before Clustered Plugin Sync Test - -The next clustered test should not need plugin disable/uninstall semantics. The important pre-flight items are: - -1. **Use the current published `guix-tribes` master baseline.** - The cluster nodes should consume `fa4753a` or a signed equivalent that includes the plugin substitute manifest and the `tribes_ui` Guix build fix. - -2. **Confirm substitute availability before provisioning.** - The builder should already have substitutes for `tribes`, `tribes-plugin-supertest`, `tribes-plugin-aether`, `tribes-plugin-sender`, and `ffmpeg`. Missing substitutes should abort cleanly or fail early; they should not leave nodes half-switched after a long silent build. - -3. **Handle the `guix-fork` channel-auth rewrite on the substitute builder path.** - The pinned `guix-fork` commit `906f6b2d3a4f9f80c5ad6f9e5f6369706a1a301d` is not a descendant of the old introduction commit `6f9c3cd1761f0a3f8b70223cb0e0f47e29582d90`. For the builder sync job that consumes explicit `pins/base-channels.sexp`, either allow downgrades there or refresh the channel introduction to the new signed lineage. Keep that scoped to the builder/pinned-channel path. - -4. **Run the new clustered plugin scenario.** - `cluster-plugin-rollout-sync-split-brain` now builds one Hetzner init node, plans Scaleway and OVH join nodes, materializes the two joins in one Legion graph run, rolls out `supertest`, partitions one node from the sync mesh, heals it, forces full resync, validates plugin table convergence, and rolls back all nodes. - -5. **Assert control-plane convergence before plugin writes.** - Before testing plugin data sync, capture and compare per-node: - - `cluster_nodes.list` - - `cluster_status` - - sync server/runtime state - - active node transport addresses - -6. **Exercise bidirectional plugin table sync.** - Minimal useful check: - - roll out `supertest` to all active nodes - - wait for selected/running convergence and plugin schema readiness on every node - - create a `Supertest.Case` on node A - - poll `/plugins-api/supertest/state?run_id=...` on node B until it appears - - create a linked `Supertest.Event` on node B - - poll node A until the event appears - - update/increment the case on one node and verify the other node sees the latest value - -7. **Run rollback after clustered sync.** - Roll back to the no-plugin baseline and verify on every active node: - - selected/running convergence back to baseline - - plugin API is gone - - `supertest_cases` and `supertest_events` are absent - - cluster status returns healthy after services restart - ---- - -## 5) Open Implementation Work - -### Migration lifecycle - -- [ ] Run target-generation up migrations before commit, instead of relying only on the post-switch `tribes-migrations` Shepherd startup path. -- [ ] Broaden rollback/down migration orchestration for core schema rollback, plugin version rollback, and plugin uninstall beyond the direct no-plugin baseline path. -- [ ] Add automated ordering tests for core up, plugin up dependency order, plugin down reverse dependency order, and migration failure handling. -- [ ] Model disable vs uninstall semantics when it becomes a product priority: - - disable: remove/disable runtime, keep data, no down migrations - - uninstall: remove runtime and run down migrations, with destructive data loss surfaced before confirmation - -### Local-control / rollout hardening - -- [ ] Add dedicated local-control integration tests for socket bind/readiness, status endpoint behavior, responsiveness during long jobs, and abort behavior under load. -- [ ] Treat `tribes-local-control` service-definition updates as restart/reboot-required work in planning and status output. -- [ ] Add or extend `tribes` tests around admin-management rollout response shapes and machine-readable failures. -- [ ] Keep a fresh-node regression check for generation comparison after the current `guix-tribes` baseline is published. - -### Cluster scenario coverage - -- [ ] Add management probes and snapshot capture for `cluster_nodes.list`, `cluster_status`, and sync runtime/server state. -- [ ] Upgrade `cluster-lifecycle` to assert membership convergence, not only node existence. -- [x] Add clustered `supertest` plugin rollout/sync/rollback scenario wiring. -- [ ] Run `cluster-plugin-rollout-sync-split-brain` on fresh infra and fix discovered cluster rollout/sync issues. -- [ ] Add trust-expansion replay coverage for newly joined or reactivated nodes. - -### Future metadata - -- [ ] Define plugin ABI/dependency compatibility metadata: - - host framework deps are part of the versioned Tribes plugin API/ABI - - `requires` / `provides` remain for plugin-to-plugin or optional service capabilities - - Guix plugin packaging validates plugin lock compatibility against the host lock/API metadata - - compiled plugin artifacts carry compatibility metadata - - runtime checks remain defense-in-depth for manually installed artifacts -- [ ] Optional typed Legion rollout convenience wrappers. -- [ ] Future reboot-planning metadata such as `rebootRequired` and `rebootReasons`. - ---- - -## 6) Working Notes - -- Run per-repo commands from `devenv shell` where applicable. -- Keep supertest scenarios black-box and contract-oriented. -- Prefer published `guix-tribes` commits and fresh provisioned nodes over local source-only validation. -- Capture enough per-node artifacts that a failed clustered run can be diagnosed without SSH state still being live. -- The next high-value scenario is clustered `supertest` plugin rollout with bidirectional synced table R/W, followed by rollback on every active node. diff --git a/package-lock.json b/package-lock.json index a22800f..84f1ca0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "tribes-supertest", "version": "0.1.0", "license": "BSD-2-Clause", + "bin": { + "supertest": "dist/cli.js" + }, "devDependencies": { "@types/node": "^22.19.1", "prettier": "^3.7.4", diff --git a/package.json b/package.json index aa411b1..bf62731 100644 --- a/package.json +++ b/package.json @@ -15,33 +15,18 @@ "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", + "bin": { + "supertest": "./dist/cli.js" + }, "files": [ "dist" ], "scripts": { "format": "prettier --write .", - "format:check": "prettier --check .", "typecheck": "tsc --noEmit -p tsconfig.json", - "build": "tsc -p tsconfig.build.json", - "check": "npm run typecheck", - "precommit": "npm run format:check && npm run typecheck && npm test && npm run build", - "preflight:substitutes": "node --import tsx scripts/check-substitutes.ts", + "build": "tsc -p tsconfig.build.json && chmod +x dist/cli.js", "test": "node --import tsx --test", - "scenario": "node --import tsx src/index.ts", - "scenario:list": "npm run scenario -- list", - "alias:tribes": "npm run scenario -- group tribes", - "alias:sender": "npm run scenario -- group sender", - "alias:kobold": "npm run scenario -- group kobold", - "alias:all": "npm run scenario -- group all", - "scenario:single-node-init": "npm run scenario -- single-node-init", - "scenario:manual-node-init": "npm run scenario -- manual-node-init", - "scenario:single-node-plugin-rollout-rollback": "npm run scenario -- single-node-plugin-rollout-rollback", - "scenario:single-node-sender": "npm run scenario -- single-node-sender", - "scenario:cluster-sender-fanout-reboot": "npm run scenario -- cluster-sender-fanout-reboot", - "scenario:cluster-kobold-public-private": "npm run scenario -- cluster-kobold-public-private", - "scenario:cluster-plugin-integrated-rollout": "npm run scenario -- cluster-plugin-integrated-rollout", - "scenario:cluster-plugin-rollout-sync-split-brain": "npm run scenario -- cluster-plugin-rollout-sync-split-brain", - "scenario:cluster-lifecycle": "npm run scenario -- cluster-lifecycle" + "precommit": "prettier --check . && npm run typecheck && npm test && npm run build" }, "devDependencies": { "@types/node": "^22.19.1", diff --git a/src/architecture.ts b/src/architecture.ts index 38b36f6..ef802cc 100644 --- a/src/architecture.ts +++ b/src/architecture.ts @@ -96,3 +96,9 @@ export interface Scenario { blocks: Record roleDefaults?: Record } + +export interface ScenarioGroup { + name: string + description: string + scenarios: string[] +} diff --git a/src/cli.ts b/src/cli.ts new file mode 100755 index 0000000..a11f0ad --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,410 @@ +#!/usr/bin/env node +import { pathToFileURL } from "node:url" + +import type { BlockRequires, Scenario, ScenarioGroup, TestBlock, Topology } from "./architecture.js" +import { buildRunId, resolveRuntimeConfig } from "./config.js" +import { blockRegistry, topologyRegistry } from "./registries.js" +import { runNamedScenario, type ScenarioRunOptions } from "./runner.js" +import { + groupDefinitions, + resolveScenario, + resolveScenarioGroup, + resolveScenarioGroupMembers, + scenarioDefinitions, + type ScenarioDefinition +} from "./scenarios.js" + +interface CliFlags { + json: boolean + keepNodes: boolean + waitOnFailure: boolean +} + +const EMPTY_FLAGS: CliFlags = { + json: false, + keepNodes: false, + waitOnFailure: false +} + +function printHelp(): void { + console.log(`Usage: + supertest scenario list [--json] + supertest scenario run [--keep-nodes] [--wait-on-failure] + supertest topology list [--json] + supertest topology run [--keep-nodes] [--wait-on-failure] + supertest block list [--json] + supertest group run [--keep-nodes] [--wait-on-failure] + supertest help | --help | -h +`) + process.stdout.write(summarizeGroups(groupDefinitions)) + console.log("") + process.stdout.write(summarizeScenarios(scenarioDefinitions)) + console.log("") + process.stdout.write(summarizeTopologies([...topologyRegistry.list()])) + console.log("") + process.stdout.write(summarizeBlocks([...blockRegistry.list()])) +} + +function printJson(value: unknown): void { + console.log(JSON.stringify(value, null, 2)) +} + +function summarizeGroups(groups: readonly ScenarioGroup[]): string { + const lines = ["Groups:"] + for (const group of groups) { + lines.push(` ${group.name}: ${group.scenarios.join(", ")}`) + lines.push(` ${group.description}`) + } + return `${lines.join("\n")}\n` +} + +function summarizeScenarios(definitions: readonly Scenario[]): string { + const lines = ["Scenarios:"] + definitions.forEach((definition, index) => { + if (index > 0) { + lines.push("") + } + + const topology = topologyRegistry.get(definition.topology) + lines.push( + ` ${definition.name} (topology ${definition.topology}, ETA ${topology?.estimatedDuration ?? "unknown"})` + ) + lines.push(` ${definition.description}`) + + const bindings = Object.entries(definition.blocks) + if (bindings.length === 0) { + lines.push(" Blocks: none (topology gates only)") + } else { + lines.push(" Blocks:") + for (const [hookName, blockNames] of bindings) { + lines.push(` ${hookName}: ${blockNames.join(", ")}`) + } + } + }) + return `${lines.join("\n")}\n` +} + +function summarizeTopologies(topologies: readonly Topology[]): string { + const lines = ["Topologies:"] + topologies.forEach((topology, index) => { + if (index > 0) { + lines.push("") + } + + lines.push(` ${topology.name} (ETA ${topology.estimatedDuration})`) + lines.push(` ${topology.description}`) + lines.push(` Roles: ${topology.roles.join(", ") || "none"}`) + lines.push(" Hooks:") + for (const hook of topology.hooks) { + const gates = hook.gates?.map((gate) => gate.name).join(", ") || "none" + lines.push( + ` ${hook.name}: minNodes=${hook.guarantees.minNodes}; roles=${hook.guarantees.roles.join(", ") || "none"}; gates=${gates}` + ) + } + }) + return `${lines.join("\n")}\n` +} + +function summarizeBlocks(blocks: readonly TestBlock[]): string { + const lines = ["Blocks:"] + blocks.forEach((block, index) => { + if (index > 0) { + lines.push("") + } + + lines.push(` ${block.name}`) + lines.push(` ${block.description}`) + lines.push(` Requires: ${formatRequires(block.requires)}`) + }) + return `${lines.join("\n")}\n` +} + +function formatRequires(requires: BlockRequires | undefined): string { + if (!requires) { + return "none" + } + + const parts: string[] = [] + if (requires.minNodes !== undefined) { + parts.push(`minNodes=${requires.minNodes}`) + } + if (requires.roles?.length) { + parts.push(`roles=${requires.roles.join(",")}`) + } + if (requires.providers?.length) { + parts.push(`providers=${requires.providers.join(",")}`) + } + return parts.join("; ") || "none" +} + +function commandArgs(ref: string | undefined, rest: string[]): string[] { + return ref ? [ref, ...rest] : rest +} + +function parseFlags( + args: string[], + allowed: readonly string[] +): { positionals: string[]; flags: CliFlags } { + const flags = { ...EMPTY_FLAGS } + const positionals: string[] = [] + const allowedSet = new Set(allowed) + + for (const arg of args) { + if (!arg.startsWith("--")) { + positionals.push(arg) + continue + } + + if (!allowedSet.has(arg)) { + throw new Error(`unknown flag for command: ${arg}`) + } + + if (arg === "--json") { + flags.json = true + } else if (arg === "--keep-nodes") { + flags.keepNodes = true + } else if (arg === "--wait-on-failure") { + flags.waitOnFailure = true + } + } + + return { positionals, flags } +} + +function runEnv(flags: CliFlags): NodeJS.ProcessEnv { + const env = { ...process.env } + if (flags.keepNodes) { + env["SUPERTEST_KEEP_NODES"] = "1" + } + if (flags.waitOnFailure) { + env["SUPERTEST_WAIT_ON_FAILURE"] = "1" + } + return env +} + +export function resolveGroupScenarioConfig( + scenarioName: string, + scenarioIndex: number, + env: NodeJS.ProcessEnv = process.env, + cwd = process.cwd(), + now = new Date() +) { + const scenarioEnv = { ...env } + + if (!scenarioEnv["SUPERTEST_RUN_ID"]) { + scenarioEnv["SUPERTEST_RUN_ID"] = buildRunId(new Date(now.getTime() + scenarioIndex)) + } + + return resolveRuntimeConfig(scenarioName, scenarioEnv, cwd, now) +} + +export function resolveGroupScenarioRunOptions( + scenarioIndex: number, + scenarioCount: number, + keepNodesRequested: boolean +): ScenarioRunOptions { + if (!keepNodesRequested) { + return {} + } + + return { + keepNodesOnSuccess: scenarioIndex === scenarioCount - 1, + keepNodesOnFailure: true + } +} + +async function runScenario(ref: string, flags: CliFlags): Promise { + const definition = resolveScenario(ref) + if (!definition) { + console.error(`unknown scenario: ${ref}`) + return 1 + } + + const config = resolveRuntimeConfig(definition.name, runEnv(flags)) + await runNamedScenario(config, definition) + return 0 +} + +async function runTopology(ref: string, flags: CliFlags): Promise { + const topology = topologyRegistry.resolveUniquePrefix(ref) + if (!topology) { + console.error(`unknown topology: ${ref}`) + return 1 + } + + const definition: ScenarioDefinition = { + name: `topology-${topology.name}`, + description: `Run ${topology.name} topology gates without scenario blocks.`, + topology: topology.name, + blocks: {} + } + const config = resolveRuntimeConfig(definition.name, runEnv(flags)) + await runNamedScenario(config, definition) + return 0 +} + +async function runGroup(ref: string, flags: CliFlags): Promise { + const group = resolveScenarioGroup(ref) + if (!group) { + console.error(`unknown group: ${ref}`) + return 1 + } + + const definitions = resolveScenarioGroupMembers(group) + if (definitions.length === 0) { + console.error(`group ${group.name} has no scenarios to run`) + return 1 + } + + console.log(`### group ${group.name} (${definitions.length} scenarios)`) + for (const definition of definitions) { + console.log(`- ${definition.name}`) + } + + const completed: ScenarioDefinition[] = [] + const env = runEnv(flags) + const keepNodesRequested = + flags.keepNodes || isTruthyEnvValue(process.env["SUPERTEST_KEEP_NODES"]) + + for (let index = 0; index < definitions.length; index += 1) { + const definition = definitions[index]! + const config = resolveGroupScenarioConfig(definition.name, index, env) + const options = resolveGroupScenarioRunOptions(index, definitions.length, keepNodesRequested) + try { + await runNamedScenario(config, definition, options) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + console.error(`\ngroup ${group.name} failed at scenario ${definition.name}: ${message}`) + console.error( + `passed before failure: ${completed.length ? completed.map((entry) => entry.name).join(", ") : "none"}` + ) + return 1 + } + completed.push(definition) + } + + console.log( + `\ngroup ${group.name} passed all ${completed.length} scenarios: ${completed.map((entry) => entry.name).join(", ")}` + ) + return 0 +} + +function isTruthyEnvValue(value: string | undefined): boolean { + return ["1", "true", "yes", "on"].includes(value?.toLowerCase() ?? "") +} + +function scenarioList(flags: CliFlags): number { + if (flags.json) { + printJson(scenarioDefinitions) + } else { + process.stdout.write(summarizeScenarios(scenarioDefinitions)) + } + return 0 +} + +function topologyList(flags: CliFlags): number { + const topologies = [...topologyRegistry.list()] + if (flags.json) { + printJson(topologies) + } else { + process.stdout.write(summarizeTopologies(topologies)) + } + return 0 +} + +function blockList(flags: CliFlags): number { + const blocks = [...blockRegistry.list()] + if (flags.json) { + printJson(blocks) + } else { + process.stdout.write(summarizeBlocks(blocks)) + } + return 0 +} + +export async function main(args: string[] = process.argv.slice(2)): Promise { + const [scope, action, ref, ...rest] = args + + if (!scope || scope === "help" || scope === "--help" || scope === "-h") { + printHelp() + return 0 + } + + if (scope === "scenario" && action === "list") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), ["--json"]) + if (positionals.length > 0) { + throw new Error( + `scenario list does not accept positional arguments: ${positionals.join(" ")}` + ) + } + return scenarioList(flags) + } + + if (scope === "scenario" && action === "run") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), [ + "--keep-nodes", + "--wait-on-failure" + ]) + const scenarioRef = positionals[0] + if (!scenarioRef || positionals.length > 1) { + throw new Error("scenario run requires exactly one scenario name") + } + return await runScenario(scenarioRef, flags) + } + + if (scope === "topology" && action === "list") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), ["--json"]) + if (positionals.length > 0) { + throw new Error( + `topology list does not accept positional arguments: ${positionals.join(" ")}` + ) + } + return topologyList(flags) + } + + if (scope === "topology" && action === "run") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), [ + "--keep-nodes", + "--wait-on-failure" + ]) + const topologyRef = positionals[0] + if (!topologyRef || positionals.length > 1) { + throw new Error("topology run requires exactly one topology name") + } + return await runTopology(topologyRef, flags) + } + + if (scope === "block" && action === "list") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), ["--json"]) + if (positionals.length > 0) { + throw new Error(`block list does not accept positional arguments: ${positionals.join(" ")}`) + } + return blockList(flags) + } + + if (scope === "group" && action === "run") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), [ + "--keep-nodes", + "--wait-on-failure" + ]) + const groupRef = positionals[0] + if (!groupRef || positionals.length > 1) { + throw new Error("group run requires exactly one group name") + } + return await runGroup(groupRef, flags) + } + + throw new Error(`unknown command: ${args.join(" ")}`) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + void main().then( + (exitCode) => { + process.exitCode = exitCode + }, + (error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } + ) +} diff --git a/src/config.ts b/src/config.ts index 5f69458..50f728d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,9 +1,6 @@ import crypto from "node:crypto" import { resolve } from "node:path" -import { topologyRegistry } from "./registries.js" -import type { ScenarioDefinition } from "./scenarios.js" - export type ProviderKind = "hetzner" | "ovh" | "scaleway" | "manual" export type LegionTestCertificateMode = "acme" | "self-signed" @@ -289,33 +286,6 @@ export function assertProviderEnvironmentAvailable(kind: ProviderKind): void { } } -export function summarizeScenarioDefinitions(definitions: ScenarioDefinition[]): string { - const lines = ["Scenarios:"] - definitions.forEach((definition, scenarioIndex) => { - if (scenarioIndex > 0) { - lines.push("") - } - - const topology = topologyRegistry.get(definition.topology) - lines.push( - ` ${definition.name} (topology ${definition.topology}, ETA ${topology?.estimatedDuration ?? "unknown"})` - ) - lines.push(` ${definition.description}`) - - const bindings = Object.entries(definition.blocks) - if (bindings.length > 0) { - lines.push(" Blocks:") - for (const [hookName, blockNames] of bindings) { - lines.push(` ${hookName}: ${blockNames.join(", ")}`) - } - } else { - lines.push(" Blocks: none (topology gates only)") - } - }) - - return `${lines.join("\n")}\n` -} - export function buildRunId(now: Date): string { const iso = now.toISOString().replaceAll(":", "").replaceAll(".", "").toLowerCase() return iso.replace("t", "t").replace("z", "z") diff --git a/src/index.ts b/src/index.ts index d798cbe..c5a7815 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,177 +1 @@ -import { pathToFileURL } from "node:url" - -import { buildRunId, resolveRuntimeConfig, summarizeScenarioDefinitions } from "./config.js" -import { - getScenarioDefinition, - listAliasNames, - resolveScenarioAlias, - scenarioAliasGroups, - scenarioDefinitions, - type ScenarioDefinition -} from "./scenarios.js" -import { runNamedScenario, type ScenarioRunOptions } from "./runner.js" - -const scenarioNames = scenarioDefinitions.map((definition) => definition.name).join("|") -const aliasNames = listAliasNames().join("|") - -function printUsage(): void { - console.log(`Usage: tribes-supertest |${scenarioNames}>`) - console.log("") - console.log("Commands:") - console.log(" list List concrete scenarios.") - console.log(" group Run an alias group of scenarios sequentially (fail-fast).") - console.log("") - process.stdout.write(summarizeAliasGroups()) - console.log("") - process.stdout.write(summarizeScenarioDefinitions(scenarioDefinitions)) -} - -function printScenarios(): void { - process.stdout.write(summarizeScenarioDefinitions(scenarioDefinitions)) -} - -function summarizeAliasGroups(): string { - const lines = ["Alias groups:"] - for (const alias of listAliasNames()) { - const definitions = resolveScenarioAlias(alias) ?? [] - lines.push(` ${alias}: ${definitions.map((definition) => definition.name).join(", ")}`) - } - return `${lines.join("\n")}\n` -} - -export function resolveGroupScenarioConfig( - scenarioName: string, - scenarioIndex: number, - env: NodeJS.ProcessEnv = process.env, - cwd = process.cwd(), - now = new Date() -) { - const scenarioEnv = { ...env } - - if (!scenarioEnv["SUPERTEST_RUN_ID"]) { - scenarioEnv["SUPERTEST_RUN_ID"] = buildRunId(new Date(now.getTime() + scenarioIndex)) - } - - return resolveRuntimeConfig(scenarioName, scenarioEnv, cwd, now) -} - -export function resolveGroupScenarioRunOptions( - scenarioIndex: number, - scenarioCount: number, - keepNodesRequested: boolean -): ScenarioRunOptions { - if (!keepNodesRequested) { - return {} - } - - return { - keepNodesOnSuccess: scenarioIndex === scenarioCount - 1, - keepNodesOnFailure: true - } -} - -async function runScenarioGroup(alias: string): Promise { - const definitions = resolveScenarioAlias(alias) - if (!definitions) { - console.error(`unknown alias: ${alias}`) - console.error("") - printUsage() - return 1 - } - - if (definitions.length === 0) { - console.error(`alias ${alias} has no scenarios to run`) - return 1 - } - - console.log(`### alias ${alias} (${definitions.length} scenarios)`) - for (const definition of definitions) { - console.log(`- ${definition.name}`) - } - - const completed: ScenarioDefinition[] = [] - for (let index = 0; index < definitions.length; index += 1) { - const definition = definitions[index]! - const config = resolveGroupScenarioConfig(definition.name, index) - const options = resolveGroupScenarioRunOptions( - index, - definitions.length, - isTruthyEnvValue(process.env["SUPERTEST_KEEP_NODES"]) - ) - try { - await runNamedScenario(config, definition, options) - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - console.error(`\nalias ${alias} failed at scenario ${definition.name}: ${message}`) - console.error( - `passed before failure: ${completed.length ? completed.map((entry) => entry.name).join(", ") : "none"}` - ) - return 1 - } - completed.push(definition) - } - - console.log( - `\nalias ${alias} passed all ${completed.length} scenarios: ${completed.map((entry) => entry.name).join(", ")}` - ) - return 0 -} - -function isTruthyEnvValue(value: string | undefined): boolean { - return ["1", "true", "yes", "on"].includes(value?.toLowerCase() ?? "") -} - -export async function main(args: string[] = process.argv.slice(2)): Promise { - const [command] = args - - if (!command || command === "help" || command === "--help" || command === "-h") { - printUsage() - return 0 - } - - if (command === "list") { - printScenarios() - return 0 - } - - if (command === "group") { - const alias = args[1] - if (!alias) { - console.error("group requires an alias name") - console.error("") - printUsage() - return 1 - } - return await runScenarioGroup(alias) - } - - const definition = getScenarioDefinition(command) - if (definition) { - const config = resolveRuntimeConfig(command) - await runNamedScenario(config, definition) - return 0 - } - - // Allow bare alias names (e.g. `tribes`, `sender`, `all`) as a shorthand for - // `group `. - if (command === "all" || command in scenarioAliasGroups) { - return await runScenarioGroup(command) - } - - console.error(`unknown command: ${command}`) - console.error("") - printUsage() - return 1 -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - void main().then( - (exitCode) => { - process.exitCode = exitCode - }, - (error: unknown) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } - ) -} +export { main, resolveGroupScenarioConfig, resolveGroupScenarioRunOptions } from "./cli.js" diff --git a/src/registries.ts b/src/registries.ts index 3cf818a..76de29d 100644 --- a/src/registries.ts +++ b/src/registries.ts @@ -1,4 +1,4 @@ -import type { Scenario, TestBlock, Topology } from "./architecture.js" +import type { Scenario, ScenarioGroup, TestBlock, Topology } from "./architecture.js" export interface NamedRegistryEntry { name: string @@ -82,3 +82,4 @@ export function defineRegistry( export const topologyRegistry = defineRegistry("topology") export const blockRegistry = defineRegistry("block") export const scenarioRegistry = defineRegistry("scenario") +export const groupRegistry = defineRegistry("group") diff --git a/src/scenarios.ts b/src/scenarios.ts index 5be572c..db9bd24 100644 --- a/src/scenarios.ts +++ b/src/scenarios.ts @@ -1,5 +1,5 @@ -import type { Scenario } from "./architecture.js" -import { blockRegistry, scenarioRegistry, topologyRegistry, type Registry } from "./registries.js" +import type { Scenario, ScenarioGroup } from "./architecture.js" +import { blockRegistry, groupRegistry, scenarioRegistry, topologyRegistry } from "./registries.js" import { clusterLifecycleScenario } from "./scenarios/cluster-lifecycle.js" import { singleNodeInitScenario } from "./scenarios/single-node-init.js" import { singleNodeInitBlocks } from "./test-blocks/single-node-init.js" @@ -7,49 +7,47 @@ import { clusterLifecycleTopology } from "./topologies/cluster-lifecycle.js" import { singleNodeTopology } from "./topologies/single-node.js" export type ScenarioDefinition = Scenario +export type GroupDefinition = ScenarioGroup topologyRegistry.registerAll([singleNodeTopology, clusterLifecycleTopology]) blockRegistry.registerAll(singleNodeInitBlocks) scenarioRegistry.registerAll([singleNodeInitScenario, clusterLifecycleScenario]) +const primaryGroups: ScenarioGroup[] = [ + { + name: "tribes", + description: + "Reference Tribes coverage across the single-node and cluster lifecycle topologies.", + scenarios: ["single-node-init", "cluster-lifecycle"] + } +] + +groupRegistry.registerAll([ + ...primaryGroups, + { + name: "all", + description: "Deduplicated union of every named group.", + scenarios: [...new Set(primaryGroups.flatMap((group) => group.scenarios))] + } +]) + export const scenarioDefinitions: ScenarioDefinition[] = [...scenarioRegistry.list()] +export const groupDefinitions: GroupDefinition[] = [...groupRegistry.list()] -export function getScenarioDefinition(name: string): ScenarioDefinition | undefined { - return scenarioRegistry.get(name) +export function resolveScenario(ref: string): ScenarioDefinition | undefined { + return scenarioRegistry.resolveUniquePrefix(ref) } -export const scenarioAliasGroups: Record = { - tribes: ["single-node-init", "cluster-lifecycle"] +export function resolveScenarioGroup(ref: string): GroupDefinition | undefined { + return groupRegistry.resolveUniquePrefix(ref) } -export const ALIAS_ALL = "all" - -export function listAliasNames(): string[] { - return [...Object.keys(scenarioAliasGroups), ALIAS_ALL] -} - -export function resolveScenarioAlias(alias: string): ScenarioDefinition[] | undefined { - if (alias === ALIAS_ALL) { - const grouped = new Set(Object.values(scenarioAliasGroups).flat()) - return scenarioDefinitions.filter((definition) => grouped.has(definition.name)) - } - - const names = scenarioAliasGroups[alias] - if (!names) { - return undefined - } - - return names.map((name) => requireScenario(scenarioRegistry, alias, name)) -} - -function requireScenario( - registry: Registry, - alias: string, - name: string -): ScenarioDefinition { - const definition = registry.get(name) - if (!definition) { - throw new Error(`alias ${alias} references unknown scenario ${name}`) - } - return definition +export function resolveScenarioGroupMembers(group: GroupDefinition): ScenarioDefinition[] { + return group.scenarios.map((name) => { + const definition = scenarioRegistry.get(name) + if (!definition) { + throw new Error(`group ${group.name} references unknown scenario ${name}`) + } + return definition + }) } diff --git a/tests/cli.test.ts b/tests/cli.test.ts new file mode 100644 index 0000000..9dadc7a --- /dev/null +++ b/tests/cli.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { main } from "../src/cli.js" + +test("scenario list is generated from registered scenarios", async () => { + const result = await captureStdout(() => main(["scenario", "list"])) + + assert.equal(result.exitCode, 0) + assert.match(result.stdout, /single-node-init/) + assert.match(result.stdout, /cluster-lifecycle/) +}) + +test("topology list supports JSON output", async () => { + const result = await captureStdout(() => main(["topology", "list", "--json"])) + const topologies = JSON.parse(result.stdout) as Array<{ name: string }> + + assert.equal(result.exitCode, 0) + assert.deepEqual( + topologies.map((topology) => topology.name), + ["single-node", "cluster-lifecycle"] + ) +}) + +test("block list is generated from registered blocks", async () => { + const result = await captureStdout(() => main(["block", "list"])) + + assert.equal(result.exitCode, 0) + assert.match(result.stdout, /http3-health/) + assert.match(result.stdout, /log-roundtrip/) +}) + +async function captureStdout( + run: () => Promise +): Promise<{ exitCode: number; stdout: string }> { + const writes: string[] = [] + const originalWrite = process.stdout.write + process.stdout.write = ((chunk: string | Uint8Array) => { + writes.push(chunk.toString()) + return true + }) as typeof process.stdout.write + + try { + const exitCode = await run() + return { exitCode, stdout: writes.join("") } + } finally { + process.stdout.write = originalWrite + } +} diff --git a/tests/index.test.ts b/tests/index.test.ts index 536f59c..a703368 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import test from "node:test" -import { resolveGroupScenarioConfig, resolveGroupScenarioRunOptions } from "../src/index.js" +import { resolveGroupScenarioConfig, resolveGroupScenarioRunOptions } from "../src/cli.js" test("group scenarios get distinct generated run metadata", () => { const env = {