diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5564ea6..36cddc6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: npm-format-check name: npm format check - entry: npm run format:check + entry: npx prettier --check . language: system pass_filenames: false files: '\.(cjs|js|json|md|mjs|ts|tsx|yaml|yml)$' diff --git a/README.md b/README.md index d7e4949..5aa0d6d 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ npm run build Check the Guix substitute servers before spending cloud time: ```bash -npm run preflight:substitutes -npm run preflight:substitutes -- --plugin sender +node --import tsx scripts/check-substitutes.ts +node --import tsx scripts/check-substitutes.ts --plugin sender ``` Delete leftover cloud resources without using Legion state: @@ -76,105 +76,39 @@ scripts/cleanup-cloud-resources --dry-run ## Basic Usage -List scenarios: +Build the CLI, then list registered scenarios, topologies, and blocks: ```bash -npm run scenario:list +npm run build +./dist/cli.js scenario list +./dist/cli.js topology list +./dist/cli.js block list ``` -Run the single-node scenario: +Run a scenario or a topology directly: ```bash -npm run scenario:single-node-init -``` - -Run the manual single-node scenario against an existing host: - -```bash -SUPERTEST_MANUAL_HOST_IP=203.0.113.10 \ - SUPERTEST_MANUAL_USERNAME=ubuntu \ - SUPERTEST_MANUAL_PASSWORD=secret \ - npm run scenario:manual-node-init -``` - -Run the single-node plugin rollout/rollback scenario: - -```bash -npm run scenario:single-node-plugin-rollout-rollback -``` - -Run the single-node Sender ingest/HLS scenario: - -```bash -npm run scenario:single-node-sender -``` - -Run the clustered Sender fanout/reboot scenario: - -```bash -npm run scenario:cluster-sender-fanout-reboot -``` - -Run the cluster lifecycle scenario: - -```bash -npm run scenario:cluster-lifecycle -``` - -Run the clustered plugin sync split-brain scenario: - -```bash -npm run scenario:cluster-plugin-rollout-sync-split-brain +./dist/cli.js scenario run single-node-init +./dist/cli.js topology run cluster-lifecycle ``` Keep created nodes around for inspection: ```bash -SUPERTEST_KEEP_NODES=1 npm run scenario:cluster-lifecycle +./dist/cli.js scenario run cluster-lifecycle --keep-nodes ``` -Run one scenario directly with the generic entrypoint: +Groups run several scenarios sequentially and fail fast on the first failing +scenario. All scenarios in a group share one `SUPERTEST_RUN_ID`, so their +artifacts land in sibling directories under the same run. ```bash -npm run scenario -- single-node-init -npm run scenario -- manual-node-init -npm run scenario -- single-node-plugin-rollout-rollback -npm run scenario -- single-node-sender -npm run scenario -- cluster-sender-fanout-reboot -npm run scenario -- cluster-plugin-rollout-sync-split-brain -npm run scenario -- cluster-lifecycle +./dist/cli.js group run tribes +./dist/cli.js group run all ``` -## Alias Groups - -Aliases run several scenarios sequentially and fail fast on the first failing -scenario. All scenarios in an alias share one `SUPERTEST_RUN_ID`, so their -artifacts land in sibling directories under the same run. Each group is the -minimal set of scenarios that preserves its coverage. - -```bash -npm run alias:tribes # Tribes core: no-plugin + supertest scenarios -npm run alias:sender # Sender ingest/HLS scenarios -npm run alias:kobold # Trust-backed Kobold dataset scenarios -npm run alias:all # deduplicated union of every named group -``` - -- `tribes` — Tribes core: `single-node-init`, - `cluster-plugin-integrated-rollout`, `cluster-plugin-rollout-sync-split-brain`, - `cluster-lifecycle`. `single-node-plugin-rollout-rollback` is omitted because - its coverage is a subset of the cluster rollout scenarios (its rollback - executor-status check is folded into `cluster-plugin-rollout-sync-split-brain`); - run it on its own with `npm run scenario:single-node-plugin-rollout-rollback`. -- `sender` — `single-node-sender`, `cluster-sender-fanout-reboot` (both kept; the - single-node run uniquely covers external endpoint types, the direct Vinyl HLS - metric, audio ingest, and a Hetzner-init origin). -- `kobold` — `cluster-kobold-public-private`. -- `all` — the deduplicated union of the named groups. This excludes - `manual-node-init` (needs a manually supplied host, `SUPERTEST_MANUAL_*`) and - the redundant `single-node-plugin-rollout-rollback`. - -Equivalent to `npm run scenario -- group `. See `npm run scenario` -(no arguments) for the current grouping. +See `./dist/cli.js help` for generated listings and the current group +membership. ## Dev Branch Helper diff --git a/docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md b/docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md deleted file mode 100644 index 0a7aee8..0000000 --- a/docs/ROLLOUT_CROSS_REPO_PLAN_PROGRESS.md +++ /dev/null @@ -1,190 +0,0 @@ -# Plugin Rollout/Rollback Cross-Repo Progress - -**Status:** single-node `supertest` plugin rollout/rollback is green on signed `guix-tribes` `master`; next target is clustered plugin sync validation. -**Last updated:** 2026-05-03 -**Scope repos:** `tribes-supertest`, `tribes-plugin-supertest`, `legion_kk`, `tribes`, `guix-tribes` - ---- - -## 1) Current Baseline - -### Green live validation - -`single-node-plugin-rollout-rollback` has passed on fresh infra through the public admin API using the real pinned `supertest` plugin: - -- run: `2026-05-01t133844554z-single-node-plugin-rollout-rollback` -- node: `st-20260501-820549-hx-a` -- consumed `guix-tribes` commit: `5a348e7c5427b99c84755aa12c30c37c2de7a4ca` -- baseline selected/running: `/gnu/store/39hsfwaf39h5h91gwd80ildrsjsdr85b-system` -- plugin selected/running: `/gnu/store/w51f5b2a5ynrpfky6qyawbnjhmgx0mxs-system` -- rollback selected/running: `/gnu/store/39hsfwaf39h5h91gwd80ildrsjsdr85b-system` -- validation: plugin API health/schema/write/read passed -- rollback validation: `supertest_events` and `supertest_cases` were removed by down migrations -- cleanup: Hetzner server destroyed; final node list `[]` - -### Current `guix-tribes` master baseline - -`guix-tribes` `master` has moved beyond the first green run and now includes the follow-up fixes and plugin substitute baseline: - -- `39b1ed8 fix: skip no-op pulls and stabilize generation diagnostics` -- `8849107 fix: resolve herd for rollback migrations` -- `e13c136 test: harden local-control worker state` -- `2950278 chore: Bump tribes` -- `05c493b test: avoid running guile suites on import` -- `0b4d3a7 fix: compile bundled tribes_ui plugin` -- `fa4753a feat: build channel plugins in substitute baseline` - -`fa4753a` is the current intended `guix-tribes` master baseline for the next test runs. The substitute manifest now covers the channel plugin registry packages, including `tribes-plugin-supertest`, `tribes-plugin-aether`, and `tribes-plugin-sender` plus `ffmpeg` for sender. - -### Current plugin fixture - -`tribes-plugin-supertest` commit `e042f3265db7a40d4d558132800238c6d466e8dd` provides: - -- `Supertest.Case` and `Supertest.Event` -- reversible migrations for `supertest_cases` and `supertest_events` -- `AshNostrSync` registration for both resources -- a local-only JSON API under `/plugins-api/supertest` - -The API is enough for black-box clustered checks because the runner can call it over SSH on each node's `127.0.0.1:4000`. - ---- - -## 2) Locked Decisions - -- Scenario assertions use the normal operator/automation surface: `POST /api/admin/management` with NIP-98 auth. -- `tribes-deploy-exec` remains diagnostic-only for these scenarios. -- `supertest` is the lean migration-bearing rollout fixture; `aether` and `sender` remain real packaged plugins for broader package/substitute coverage. -- Selected generation is `/var/guix/profiles/system`; running generation is `/run/current-system`. -- Guix-sensitive work must run through the pulled/current Guix profile/module universe. -- Direct explicit generation rollback must stay async/restart-tolerant and must not replay the current rollout plan. -- `tribes-local-control` service-definition updates should be treated as restart/reboot-required work, not restarted in-band while local-control is serving the switch/rollback request. -- Plugin disable is lower priority than clustered install/sync/rollback validation. - ---- - -## 3) What Works Now - -### `tribes-supertest` - -- Public-admin-API rollout flow for `single-node-plugin-rollout-rollback`. -- Rollout assertions for preview/start/status, selected/running convergence, plugin API health/schema/write/read, rollback convergence, table removal, and provider cleanup. -- Strong diagnostics around local-control readiness, socket/listener readiness, migrations, Shepherd logs, certificates, system generation comparison, Guix build logs, and daemon logs. - -### `guix-tribes` - -- External plugin pinning and registry-backed plugin packaging. -- Target-node plugin builds with the lean Mix/OTP baseline. -- Runtime plugin loading in release mode. -- `tribes-migrations` wrapper logging with real output/exit status. -- Current-Guix delegation for service upgrade, generation comparison, config evaluation, and derivation realization. -- Rollback to installed baseline profile generations. -- Plugin rollback down migrations for direct rollback from a plugin generation to a no-plugin baseline. -- Target-generation plugin state restore before rollback. -- No-op `guix pull` skip when rollout plans explicitly resolve no channel delta. -- Generation diagnostic JSON stability. -- Channel plugin substitute manifest coverage. - -### `tribes` - -- Public admin-management rollout methods are the scenario contract. -- Startup readiness returns `503` while `Tribes.Repo` is unavailable. -- Explicit rollback generation is async and does not replay the current rollout plan. -- AshNostrSync publishes and consumes host and plugin resources through registered namespaces. - -### `legion_kk` - -- Generic admin management calls are available through Legion-owned CLI/service/API wrappers. -- Provider cleanup is materially more stable across Hetzner, Scaleway, and OVH. -- Installed-system Guix checkout-cache transplant is in place. -- Admin API retry behavior handles explicit `503` readiness responses. - ---- - -## 4) Remaining Gaps Before Clustered Plugin Sync Test - -The next clustered test should not need plugin disable/uninstall semantics. The important pre-flight items are: - -1. **Use the current published `guix-tribes` master baseline.** - The cluster nodes should consume `fa4753a` or a signed equivalent that includes the plugin substitute manifest and the `tribes_ui` Guix build fix. - -2. **Confirm substitute availability before provisioning.** - The builder should already have substitutes for `tribes`, `tribes-plugin-supertest`, `tribes-plugin-aether`, `tribes-plugin-sender`, and `ffmpeg`. Missing substitutes should abort cleanly or fail early; they should not leave nodes half-switched after a long silent build. - -3. **Handle the `guix-fork` channel-auth rewrite on the substitute builder path.** - The pinned `guix-fork` commit `906f6b2d3a4f9f80c5ad6f9e5f6369706a1a301d` is not a descendant of the old introduction commit `6f9c3cd1761f0a3f8b70223cb0e0f47e29582d90`. For the builder sync job that consumes explicit `pins/base-channels.sexp`, either allow downgrades there or refresh the channel introduction to the new signed lineage. Keep that scoped to the builder/pinned-channel path. - -4. **Run the new clustered plugin scenario.** - `cluster-plugin-rollout-sync-split-brain` now builds one Hetzner init node, plans Scaleway and OVH join nodes, materializes the two joins in one Legion graph run, rolls out `supertest`, partitions one node from the sync mesh, heals it, forces full resync, validates plugin table convergence, and rolls back all nodes. - -5. **Assert control-plane convergence before plugin writes.** - Before testing plugin data sync, capture and compare per-node: - - `cluster_nodes.list` - - `cluster_status` - - sync server/runtime state - - active node transport addresses - -6. **Exercise bidirectional plugin table sync.** - Minimal useful check: - - roll out `supertest` to all active nodes - - wait for selected/running convergence and plugin schema readiness on every node - - create a `Supertest.Case` on node A - - poll `/plugins-api/supertest/state?run_id=...` on node B until it appears - - create a linked `Supertest.Event` on node B - - poll node A until the event appears - - update/increment the case on one node and verify the other node sees the latest value - -7. **Run rollback after clustered sync.** - Roll back to the no-plugin baseline and verify on every active node: - - selected/running convergence back to baseline - - plugin API is gone - - `supertest_cases` and `supertest_events` are absent - - cluster status returns healthy after services restart - ---- - -## 5) Open Implementation Work - -### Migration lifecycle - -- [ ] Run target-generation up migrations before commit, instead of relying only on the post-switch `tribes-migrations` Shepherd startup path. -- [ ] Broaden rollback/down migration orchestration for core schema rollback, plugin version rollback, and plugin uninstall beyond the direct no-plugin baseline path. -- [ ] Add automated ordering tests for core up, plugin up dependency order, plugin down reverse dependency order, and migration failure handling. -- [ ] Model disable vs uninstall semantics when it becomes a product priority: - - disable: remove/disable runtime, keep data, no down migrations - - uninstall: remove runtime and run down migrations, with destructive data loss surfaced before confirmation - -### Local-control / rollout hardening - -- [ ] Add dedicated local-control integration tests for socket bind/readiness, status endpoint behavior, responsiveness during long jobs, and abort behavior under load. -- [ ] Treat `tribes-local-control` service-definition updates as restart/reboot-required work in planning and status output. -- [ ] Add or extend `tribes` tests around admin-management rollout response shapes and machine-readable failures. -- [ ] Keep a fresh-node regression check for generation comparison after the current `guix-tribes` baseline is published. - -### Cluster scenario coverage - -- [ ] Add management probes and snapshot capture for `cluster_nodes.list`, `cluster_status`, and sync runtime/server state. -- [ ] Upgrade `cluster-lifecycle` to assert membership convergence, not only node existence. -- [x] Add clustered `supertest` plugin rollout/sync/rollback scenario wiring. -- [ ] Run `cluster-plugin-rollout-sync-split-brain` on fresh infra and fix discovered cluster rollout/sync issues. -- [ ] Add trust-expansion replay coverage for newly joined or reactivated nodes. - -### Future metadata - -- [ ] Define plugin ABI/dependency compatibility metadata: - - host framework deps are part of the versioned Tribes plugin API/ABI - - `requires` / `provides` remain for plugin-to-plugin or optional service capabilities - - Guix plugin packaging validates plugin lock compatibility against the host lock/API metadata - - compiled plugin artifacts carry compatibility metadata - - runtime checks remain defense-in-depth for manually installed artifacts -- [ ] Optional typed Legion rollout convenience wrappers. -- [ ] Future reboot-planning metadata such as `rebootRequired` and `rebootReasons`. - ---- - -## 6) Working Notes - -- Run per-repo commands from `devenv shell` where applicable. -- Keep supertest scenarios black-box and contract-oriented. -- Prefer published `guix-tribes` commits and fresh provisioned nodes over local source-only validation. -- Capture enough per-node artifacts that a failed clustered run can be diagnosed without SSH state still being live. -- The next high-value scenario is clustered `supertest` plugin rollout with bidirectional synced table R/W, followed by rollback on every active node. diff --git a/package-lock.json b/package-lock.json index a22800f..84f1ca0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "tribes-supertest", "version": "0.1.0", "license": "BSD-2-Clause", + "bin": { + "supertest": "dist/cli.js" + }, "devDependencies": { "@types/node": "^22.19.1", "prettier": "^3.7.4", diff --git a/package.json b/package.json index aa411b1..bf62731 100644 --- a/package.json +++ b/package.json @@ -15,33 +15,18 @@ "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", + "bin": { + "supertest": "./dist/cli.js" + }, "files": [ "dist" ], "scripts": { "format": "prettier --write .", - "format:check": "prettier --check .", "typecheck": "tsc --noEmit -p tsconfig.json", - "build": "tsc -p tsconfig.build.json", - "check": "npm run typecheck", - "precommit": "npm run format:check && npm run typecheck && npm test && npm run build", - "preflight:substitutes": "node --import tsx scripts/check-substitutes.ts", + "build": "tsc -p tsconfig.build.json && chmod +x dist/cli.js", "test": "node --import tsx --test", - "scenario": "node --import tsx src/index.ts", - "scenario:list": "npm run scenario -- list", - "alias:tribes": "npm run scenario -- group tribes", - "alias:sender": "npm run scenario -- group sender", - "alias:kobold": "npm run scenario -- group kobold", - "alias:all": "npm run scenario -- group all", - "scenario:single-node-init": "npm run scenario -- single-node-init", - "scenario:manual-node-init": "npm run scenario -- manual-node-init", - "scenario:single-node-plugin-rollout-rollback": "npm run scenario -- single-node-plugin-rollout-rollback", - "scenario:single-node-sender": "npm run scenario -- single-node-sender", - "scenario:cluster-sender-fanout-reboot": "npm run scenario -- cluster-sender-fanout-reboot", - "scenario:cluster-kobold-public-private": "npm run scenario -- cluster-kobold-public-private", - "scenario:cluster-plugin-integrated-rollout": "npm run scenario -- cluster-plugin-integrated-rollout", - "scenario:cluster-plugin-rollout-sync-split-brain": "npm run scenario -- cluster-plugin-rollout-sync-split-brain", - "scenario:cluster-lifecycle": "npm run scenario -- cluster-lifecycle" + "precommit": "prettier --check . && npm run typecheck && npm test && npm run build" }, "devDependencies": { "@types/node": "^22.19.1", diff --git a/src/architecture.ts b/src/architecture.ts index 38b36f6..ef802cc 100644 --- a/src/architecture.ts +++ b/src/architecture.ts @@ -96,3 +96,9 @@ export interface Scenario { blocks: Record roleDefaults?: Record } + +export interface ScenarioGroup { + name: string + description: string + scenarios: string[] +} diff --git a/src/cli.ts b/src/cli.ts new file mode 100755 index 0000000..a11f0ad --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,410 @@ +#!/usr/bin/env node +import { pathToFileURL } from "node:url" + +import type { BlockRequires, Scenario, ScenarioGroup, TestBlock, Topology } from "./architecture.js" +import { buildRunId, resolveRuntimeConfig } from "./config.js" +import { blockRegistry, topologyRegistry } from "./registries.js" +import { runNamedScenario, type ScenarioRunOptions } from "./runner.js" +import { + groupDefinitions, + resolveScenario, + resolveScenarioGroup, + resolveScenarioGroupMembers, + scenarioDefinitions, + type ScenarioDefinition +} from "./scenarios.js" + +interface CliFlags { + json: boolean + keepNodes: boolean + waitOnFailure: boolean +} + +const EMPTY_FLAGS: CliFlags = { + json: false, + keepNodes: false, + waitOnFailure: false +} + +function printHelp(): void { + console.log(`Usage: + supertest scenario list [--json] + supertest scenario run [--keep-nodes] [--wait-on-failure] + supertest topology list [--json] + supertest topology run [--keep-nodes] [--wait-on-failure] + supertest block list [--json] + supertest group run [--keep-nodes] [--wait-on-failure] + supertest help | --help | -h +`) + process.stdout.write(summarizeGroups(groupDefinitions)) + console.log("") + process.stdout.write(summarizeScenarios(scenarioDefinitions)) + console.log("") + process.stdout.write(summarizeTopologies([...topologyRegistry.list()])) + console.log("") + process.stdout.write(summarizeBlocks([...blockRegistry.list()])) +} + +function printJson(value: unknown): void { + console.log(JSON.stringify(value, null, 2)) +} + +function summarizeGroups(groups: readonly ScenarioGroup[]): string { + const lines = ["Groups:"] + for (const group of groups) { + lines.push(` ${group.name}: ${group.scenarios.join(", ")}`) + lines.push(` ${group.description}`) + } + return `${lines.join("\n")}\n` +} + +function summarizeScenarios(definitions: readonly Scenario[]): string { + const lines = ["Scenarios:"] + definitions.forEach((definition, index) => { + if (index > 0) { + lines.push("") + } + + const topology = topologyRegistry.get(definition.topology) + lines.push( + ` ${definition.name} (topology ${definition.topology}, ETA ${topology?.estimatedDuration ?? "unknown"})` + ) + lines.push(` ${definition.description}`) + + const bindings = Object.entries(definition.blocks) + if (bindings.length === 0) { + lines.push(" Blocks: none (topology gates only)") + } else { + lines.push(" Blocks:") + for (const [hookName, blockNames] of bindings) { + lines.push(` ${hookName}: ${blockNames.join(", ")}`) + } + } + }) + return `${lines.join("\n")}\n` +} + +function summarizeTopologies(topologies: readonly Topology[]): string { + const lines = ["Topologies:"] + topologies.forEach((topology, index) => { + if (index > 0) { + lines.push("") + } + + lines.push(` ${topology.name} (ETA ${topology.estimatedDuration})`) + lines.push(` ${topology.description}`) + lines.push(` Roles: ${topology.roles.join(", ") || "none"}`) + lines.push(" Hooks:") + for (const hook of topology.hooks) { + const gates = hook.gates?.map((gate) => gate.name).join(", ") || "none" + lines.push( + ` ${hook.name}: minNodes=${hook.guarantees.minNodes}; roles=${hook.guarantees.roles.join(", ") || "none"}; gates=${gates}` + ) + } + }) + return `${lines.join("\n")}\n` +} + +function summarizeBlocks(blocks: readonly TestBlock[]): string { + const lines = ["Blocks:"] + blocks.forEach((block, index) => { + if (index > 0) { + lines.push("") + } + + lines.push(` ${block.name}`) + lines.push(` ${block.description}`) + lines.push(` Requires: ${formatRequires(block.requires)}`) + }) + return `${lines.join("\n")}\n` +} + +function formatRequires(requires: BlockRequires | undefined): string { + if (!requires) { + return "none" + } + + const parts: string[] = [] + if (requires.minNodes !== undefined) { + parts.push(`minNodes=${requires.minNodes}`) + } + if (requires.roles?.length) { + parts.push(`roles=${requires.roles.join(",")}`) + } + if (requires.providers?.length) { + parts.push(`providers=${requires.providers.join(",")}`) + } + return parts.join("; ") || "none" +} + +function commandArgs(ref: string | undefined, rest: string[]): string[] { + return ref ? [ref, ...rest] : rest +} + +function parseFlags( + args: string[], + allowed: readonly string[] +): { positionals: string[]; flags: CliFlags } { + const flags = { ...EMPTY_FLAGS } + const positionals: string[] = [] + const allowedSet = new Set(allowed) + + for (const arg of args) { + if (!arg.startsWith("--")) { + positionals.push(arg) + continue + } + + if (!allowedSet.has(arg)) { + throw new Error(`unknown flag for command: ${arg}`) + } + + if (arg === "--json") { + flags.json = true + } else if (arg === "--keep-nodes") { + flags.keepNodes = true + } else if (arg === "--wait-on-failure") { + flags.waitOnFailure = true + } + } + + return { positionals, flags } +} + +function runEnv(flags: CliFlags): NodeJS.ProcessEnv { + const env = { ...process.env } + if (flags.keepNodes) { + env["SUPERTEST_KEEP_NODES"] = "1" + } + if (flags.waitOnFailure) { + env["SUPERTEST_WAIT_ON_FAILURE"] = "1" + } + return env +} + +export function resolveGroupScenarioConfig( + scenarioName: string, + scenarioIndex: number, + env: NodeJS.ProcessEnv = process.env, + cwd = process.cwd(), + now = new Date() +) { + const scenarioEnv = { ...env } + + if (!scenarioEnv["SUPERTEST_RUN_ID"]) { + scenarioEnv["SUPERTEST_RUN_ID"] = buildRunId(new Date(now.getTime() + scenarioIndex)) + } + + return resolveRuntimeConfig(scenarioName, scenarioEnv, cwd, now) +} + +export function resolveGroupScenarioRunOptions( + scenarioIndex: number, + scenarioCount: number, + keepNodesRequested: boolean +): ScenarioRunOptions { + if (!keepNodesRequested) { + return {} + } + + return { + keepNodesOnSuccess: scenarioIndex === scenarioCount - 1, + keepNodesOnFailure: true + } +} + +async function runScenario(ref: string, flags: CliFlags): Promise { + const definition = resolveScenario(ref) + if (!definition) { + console.error(`unknown scenario: ${ref}`) + return 1 + } + + const config = resolveRuntimeConfig(definition.name, runEnv(flags)) + await runNamedScenario(config, definition) + return 0 +} + +async function runTopology(ref: string, flags: CliFlags): Promise { + const topology = topologyRegistry.resolveUniquePrefix(ref) + if (!topology) { + console.error(`unknown topology: ${ref}`) + return 1 + } + + const definition: ScenarioDefinition = { + name: `topology-${topology.name}`, + description: `Run ${topology.name} topology gates without scenario blocks.`, + topology: topology.name, + blocks: {} + } + const config = resolveRuntimeConfig(definition.name, runEnv(flags)) + await runNamedScenario(config, definition) + return 0 +} + +async function runGroup(ref: string, flags: CliFlags): Promise { + const group = resolveScenarioGroup(ref) + if (!group) { + console.error(`unknown group: ${ref}`) + return 1 + } + + const definitions = resolveScenarioGroupMembers(group) + if (definitions.length === 0) { + console.error(`group ${group.name} has no scenarios to run`) + return 1 + } + + console.log(`### group ${group.name} (${definitions.length} scenarios)`) + for (const definition of definitions) { + console.log(`- ${definition.name}`) + } + + const completed: ScenarioDefinition[] = [] + const env = runEnv(flags) + const keepNodesRequested = + flags.keepNodes || isTruthyEnvValue(process.env["SUPERTEST_KEEP_NODES"]) + + for (let index = 0; index < definitions.length; index += 1) { + const definition = definitions[index]! + const config = resolveGroupScenarioConfig(definition.name, index, env) + const options = resolveGroupScenarioRunOptions(index, definitions.length, keepNodesRequested) + try { + await runNamedScenario(config, definition, options) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + console.error(`\ngroup ${group.name} failed at scenario ${definition.name}: ${message}`) + console.error( + `passed before failure: ${completed.length ? completed.map((entry) => entry.name).join(", ") : "none"}` + ) + return 1 + } + completed.push(definition) + } + + console.log( + `\ngroup ${group.name} passed all ${completed.length} scenarios: ${completed.map((entry) => entry.name).join(", ")}` + ) + return 0 +} + +function isTruthyEnvValue(value: string | undefined): boolean { + return ["1", "true", "yes", "on"].includes(value?.toLowerCase() ?? "") +} + +function scenarioList(flags: CliFlags): number { + if (flags.json) { + printJson(scenarioDefinitions) + } else { + process.stdout.write(summarizeScenarios(scenarioDefinitions)) + } + return 0 +} + +function topologyList(flags: CliFlags): number { + const topologies = [...topologyRegistry.list()] + if (flags.json) { + printJson(topologies) + } else { + process.stdout.write(summarizeTopologies(topologies)) + } + return 0 +} + +function blockList(flags: CliFlags): number { + const blocks = [...blockRegistry.list()] + if (flags.json) { + printJson(blocks) + } else { + process.stdout.write(summarizeBlocks(blocks)) + } + return 0 +} + +export async function main(args: string[] = process.argv.slice(2)): Promise { + const [scope, action, ref, ...rest] = args + + if (!scope || scope === "help" || scope === "--help" || scope === "-h") { + printHelp() + return 0 + } + + if (scope === "scenario" && action === "list") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), ["--json"]) + if (positionals.length > 0) { + throw new Error( + `scenario list does not accept positional arguments: ${positionals.join(" ")}` + ) + } + return scenarioList(flags) + } + + if (scope === "scenario" && action === "run") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), [ + "--keep-nodes", + "--wait-on-failure" + ]) + const scenarioRef = positionals[0] + if (!scenarioRef || positionals.length > 1) { + throw new Error("scenario run requires exactly one scenario name") + } + return await runScenario(scenarioRef, flags) + } + + if (scope === "topology" && action === "list") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), ["--json"]) + if (positionals.length > 0) { + throw new Error( + `topology list does not accept positional arguments: ${positionals.join(" ")}` + ) + } + return topologyList(flags) + } + + if (scope === "topology" && action === "run") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), [ + "--keep-nodes", + "--wait-on-failure" + ]) + const topologyRef = positionals[0] + if (!topologyRef || positionals.length > 1) { + throw new Error("topology run requires exactly one topology name") + } + return await runTopology(topologyRef, flags) + } + + if (scope === "block" && action === "list") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), ["--json"]) + if (positionals.length > 0) { + throw new Error(`block list does not accept positional arguments: ${positionals.join(" ")}`) + } + return blockList(flags) + } + + if (scope === "group" && action === "run") { + const { positionals, flags } = parseFlags(commandArgs(ref, rest), [ + "--keep-nodes", + "--wait-on-failure" + ]) + const groupRef = positionals[0] + if (!groupRef || positionals.length > 1) { + throw new Error("group run requires exactly one group name") + } + return await runGroup(groupRef, flags) + } + + throw new Error(`unknown command: ${args.join(" ")}`) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + void main().then( + (exitCode) => { + process.exitCode = exitCode + }, + (error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } + ) +} diff --git a/src/config.ts b/src/config.ts index 5f69458..50f728d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,9 +1,6 @@ import crypto from "node:crypto" import { resolve } from "node:path" -import { topologyRegistry } from "./registries.js" -import type { ScenarioDefinition } from "./scenarios.js" - export type ProviderKind = "hetzner" | "ovh" | "scaleway" | "manual" export type LegionTestCertificateMode = "acme" | "self-signed" @@ -289,33 +286,6 @@ export function assertProviderEnvironmentAvailable(kind: ProviderKind): void { } } -export function summarizeScenarioDefinitions(definitions: ScenarioDefinition[]): string { - const lines = ["Scenarios:"] - definitions.forEach((definition, scenarioIndex) => { - if (scenarioIndex > 0) { - lines.push("") - } - - const topology = topologyRegistry.get(definition.topology) - lines.push( - ` ${definition.name} (topology ${definition.topology}, ETA ${topology?.estimatedDuration ?? "unknown"})` - ) - lines.push(` ${definition.description}`) - - const bindings = Object.entries(definition.blocks) - if (bindings.length > 0) { - lines.push(" Blocks:") - for (const [hookName, blockNames] of bindings) { - lines.push(` ${hookName}: ${blockNames.join(", ")}`) - } - } else { - lines.push(" Blocks: none (topology gates only)") - } - }) - - return `${lines.join("\n")}\n` -} - export function buildRunId(now: Date): string { const iso = now.toISOString().replaceAll(":", "").replaceAll(".", "").toLowerCase() return iso.replace("t", "t").replace("z", "z") diff --git a/src/index.ts b/src/index.ts index d798cbe..c5a7815 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,177 +1 @@ -import { pathToFileURL } from "node:url" - -import { buildRunId, resolveRuntimeConfig, summarizeScenarioDefinitions } from "./config.js" -import { - getScenarioDefinition, - listAliasNames, - resolveScenarioAlias, - scenarioAliasGroups, - scenarioDefinitions, - type ScenarioDefinition -} from "./scenarios.js" -import { runNamedScenario, type ScenarioRunOptions } from "./runner.js" - -const scenarioNames = scenarioDefinitions.map((definition) => definition.name).join("|") -const aliasNames = listAliasNames().join("|") - -function printUsage(): void { - console.log(`Usage: tribes-supertest |${scenarioNames}>`) - console.log("") - console.log("Commands:") - console.log(" list List concrete scenarios.") - console.log(" group Run an alias group of scenarios sequentially (fail-fast).") - console.log("") - process.stdout.write(summarizeAliasGroups()) - console.log("") - process.stdout.write(summarizeScenarioDefinitions(scenarioDefinitions)) -} - -function printScenarios(): void { - process.stdout.write(summarizeScenarioDefinitions(scenarioDefinitions)) -} - -function summarizeAliasGroups(): string { - const lines = ["Alias groups:"] - for (const alias of listAliasNames()) { - const definitions = resolveScenarioAlias(alias) ?? [] - lines.push(` ${alias}: ${definitions.map((definition) => definition.name).join(", ")}`) - } - return `${lines.join("\n")}\n` -} - -export function resolveGroupScenarioConfig( - scenarioName: string, - scenarioIndex: number, - env: NodeJS.ProcessEnv = process.env, - cwd = process.cwd(), - now = new Date() -) { - const scenarioEnv = { ...env } - - if (!scenarioEnv["SUPERTEST_RUN_ID"]) { - scenarioEnv["SUPERTEST_RUN_ID"] = buildRunId(new Date(now.getTime() + scenarioIndex)) - } - - return resolveRuntimeConfig(scenarioName, scenarioEnv, cwd, now) -} - -export function resolveGroupScenarioRunOptions( - scenarioIndex: number, - scenarioCount: number, - keepNodesRequested: boolean -): ScenarioRunOptions { - if (!keepNodesRequested) { - return {} - } - - return { - keepNodesOnSuccess: scenarioIndex === scenarioCount - 1, - keepNodesOnFailure: true - } -} - -async function runScenarioGroup(alias: string): Promise { - const definitions = resolveScenarioAlias(alias) - if (!definitions) { - console.error(`unknown alias: ${alias}`) - console.error("") - printUsage() - return 1 - } - - if (definitions.length === 0) { - console.error(`alias ${alias} has no scenarios to run`) - return 1 - } - - console.log(`### alias ${alias} (${definitions.length} scenarios)`) - for (const definition of definitions) { - console.log(`- ${definition.name}`) - } - - const completed: ScenarioDefinition[] = [] - for (let index = 0; index < definitions.length; index += 1) { - const definition = definitions[index]! - const config = resolveGroupScenarioConfig(definition.name, index) - const options = resolveGroupScenarioRunOptions( - index, - definitions.length, - isTruthyEnvValue(process.env["SUPERTEST_KEEP_NODES"]) - ) - try { - await runNamedScenario(config, definition, options) - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - console.error(`\nalias ${alias} failed at scenario ${definition.name}: ${message}`) - console.error( - `passed before failure: ${completed.length ? completed.map((entry) => entry.name).join(", ") : "none"}` - ) - return 1 - } - completed.push(definition) - } - - console.log( - `\nalias ${alias} passed all ${completed.length} scenarios: ${completed.map((entry) => entry.name).join(", ")}` - ) - return 0 -} - -function isTruthyEnvValue(value: string | undefined): boolean { - return ["1", "true", "yes", "on"].includes(value?.toLowerCase() ?? "") -} - -export async function main(args: string[] = process.argv.slice(2)): Promise { - const [command] = args - - if (!command || command === "help" || command === "--help" || command === "-h") { - printUsage() - return 0 - } - - if (command === "list") { - printScenarios() - return 0 - } - - if (command === "group") { - const alias = args[1] - if (!alias) { - console.error("group requires an alias name") - console.error("") - printUsage() - return 1 - } - return await runScenarioGroup(alias) - } - - const definition = getScenarioDefinition(command) - if (definition) { - const config = resolveRuntimeConfig(command) - await runNamedScenario(config, definition) - return 0 - } - - // Allow bare alias names (e.g. `tribes`, `sender`, `all`) as a shorthand for - // `group `. - if (command === "all" || command in scenarioAliasGroups) { - return await runScenarioGroup(command) - } - - console.error(`unknown command: ${command}`) - console.error("") - printUsage() - return 1 -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - void main().then( - (exitCode) => { - process.exitCode = exitCode - }, - (error: unknown) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } - ) -} +export { main, resolveGroupScenarioConfig, resolveGroupScenarioRunOptions } from "./cli.js" diff --git a/src/registries.ts b/src/registries.ts index 3cf818a..76de29d 100644 --- a/src/registries.ts +++ b/src/registries.ts @@ -1,4 +1,4 @@ -import type { Scenario, TestBlock, Topology } from "./architecture.js" +import type { Scenario, ScenarioGroup, TestBlock, Topology } from "./architecture.js" export interface NamedRegistryEntry { name: string @@ -82,3 +82,4 @@ export function defineRegistry( export const topologyRegistry = defineRegistry("topology") export const blockRegistry = defineRegistry("block") export const scenarioRegistry = defineRegistry("scenario") +export const groupRegistry = defineRegistry("group") diff --git a/src/scenarios.ts b/src/scenarios.ts index 5be572c..db9bd24 100644 --- a/src/scenarios.ts +++ b/src/scenarios.ts @@ -1,5 +1,5 @@ -import type { Scenario } from "./architecture.js" -import { blockRegistry, scenarioRegistry, topologyRegistry, type Registry } from "./registries.js" +import type { Scenario, ScenarioGroup } from "./architecture.js" +import { blockRegistry, groupRegistry, scenarioRegistry, topologyRegistry } from "./registries.js" import { clusterLifecycleScenario } from "./scenarios/cluster-lifecycle.js" import { singleNodeInitScenario } from "./scenarios/single-node-init.js" import { singleNodeInitBlocks } from "./test-blocks/single-node-init.js" @@ -7,49 +7,47 @@ import { clusterLifecycleTopology } from "./topologies/cluster-lifecycle.js" import { singleNodeTopology } from "./topologies/single-node.js" export type ScenarioDefinition = Scenario +export type GroupDefinition = ScenarioGroup topologyRegistry.registerAll([singleNodeTopology, clusterLifecycleTopology]) blockRegistry.registerAll(singleNodeInitBlocks) scenarioRegistry.registerAll([singleNodeInitScenario, clusterLifecycleScenario]) +const primaryGroups: ScenarioGroup[] = [ + { + name: "tribes", + description: + "Reference Tribes coverage across the single-node and cluster lifecycle topologies.", + scenarios: ["single-node-init", "cluster-lifecycle"] + } +] + +groupRegistry.registerAll([ + ...primaryGroups, + { + name: "all", + description: "Deduplicated union of every named group.", + scenarios: [...new Set(primaryGroups.flatMap((group) => group.scenarios))] + } +]) + export const scenarioDefinitions: ScenarioDefinition[] = [...scenarioRegistry.list()] +export const groupDefinitions: GroupDefinition[] = [...groupRegistry.list()] -export function getScenarioDefinition(name: string): ScenarioDefinition | undefined { - return scenarioRegistry.get(name) +export function resolveScenario(ref: string): ScenarioDefinition | undefined { + return scenarioRegistry.resolveUniquePrefix(ref) } -export const scenarioAliasGroups: Record = { - tribes: ["single-node-init", "cluster-lifecycle"] +export function resolveScenarioGroup(ref: string): GroupDefinition | undefined { + return groupRegistry.resolveUniquePrefix(ref) } -export const ALIAS_ALL = "all" - -export function listAliasNames(): string[] { - return [...Object.keys(scenarioAliasGroups), ALIAS_ALL] -} - -export function resolveScenarioAlias(alias: string): ScenarioDefinition[] | undefined { - if (alias === ALIAS_ALL) { - const grouped = new Set(Object.values(scenarioAliasGroups).flat()) - return scenarioDefinitions.filter((definition) => grouped.has(definition.name)) - } - - const names = scenarioAliasGroups[alias] - if (!names) { - return undefined - } - - return names.map((name) => requireScenario(scenarioRegistry, alias, name)) -} - -function requireScenario( - registry: Registry, - alias: string, - name: string -): ScenarioDefinition { - const definition = registry.get(name) - if (!definition) { - throw new Error(`alias ${alias} references unknown scenario ${name}`) - } - return definition +export function resolveScenarioGroupMembers(group: GroupDefinition): ScenarioDefinition[] { + return group.scenarios.map((name) => { + const definition = scenarioRegistry.get(name) + if (!definition) { + throw new Error(`group ${group.name} references unknown scenario ${name}`) + } + return definition + }) } diff --git a/tests/cli.test.ts b/tests/cli.test.ts new file mode 100644 index 0000000..9dadc7a --- /dev/null +++ b/tests/cli.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { main } from "../src/cli.js" + +test("scenario list is generated from registered scenarios", async () => { + const result = await captureStdout(() => main(["scenario", "list"])) + + assert.equal(result.exitCode, 0) + assert.match(result.stdout, /single-node-init/) + assert.match(result.stdout, /cluster-lifecycle/) +}) + +test("topology list supports JSON output", async () => { + const result = await captureStdout(() => main(["topology", "list", "--json"])) + const topologies = JSON.parse(result.stdout) as Array<{ name: string }> + + assert.equal(result.exitCode, 0) + assert.deepEqual( + topologies.map((topology) => topology.name), + ["single-node", "cluster-lifecycle"] + ) +}) + +test("block list is generated from registered blocks", async () => { + const result = await captureStdout(() => main(["block", "list"])) + + assert.equal(result.exitCode, 0) + assert.match(result.stdout, /http3-health/) + assert.match(result.stdout, /log-roundtrip/) +}) + +async function captureStdout( + run: () => Promise +): Promise<{ exitCode: number; stdout: string }> { + const writes: string[] = [] + const originalWrite = process.stdout.write + process.stdout.write = ((chunk: string | Uint8Array) => { + writes.push(chunk.toString()) + return true + }) as typeof process.stdout.write + + try { + const exitCode = await run() + return { exitCode, stdout: writes.join("") } + } finally { + process.stdout.write = originalWrite + } +} diff --git a/tests/index.test.ts b/tests/index.test.ts index 536f59c..a703368 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import test from "node:test" -import { resolveGroupScenarioConfig, resolveGroupScenarioRunOptions } from "../src/index.js" +import { resolveGroupScenarioConfig, resolveGroupScenarioRunOptions } from "../src/cli.js" test("group scenarios get distinct generated run metadata", () => { const env = {