import assert from "node:assert/strict" import { writeFile } from "node:fs/promises" import test from "node:test" import { APP_STATE_VERSION, type ClusterBootstrapState, type DeploymentActivityEvent, type ManagedNodeBootstrapState, type NodeExecutionInstallInput, type NodeReinstallRequest, type NodeExecutionState, type ProductCatalog, type ProviderConfig, type StoredState, type VpsRecord } from "../../src/shared/app" import type { NodeDeploymentInstallSession } from "../../src/main/deployment/service" import { NodeDeploymentRuntime, type NodeDeploymentPhaseRuntime } from "../../src/main/deployment/runtime" import { deriveNodePublicKey } from "../../src/main/bootstrap-crypto" import { getProviderDeploymentProfile } from "../../src/main/cloud/providers/registry" function createTestClusterBootstrap(): ClusterBootstrapState { return { rootSecret: "root-secret", systemAdmin: { username: "legion-system", nostrPublicKey: "e".repeat(64), nostrPrivateKey: "1".repeat(64) }, humanAdmin: { username: "root", passwordHash: "password-hash", nostrPublicKey: "f".repeat(64), encryptedNostrPrivateKey: "ciphertext", nostrPrivateKeyNonce: "nonce", nostrPrivateKeySalt: "salt" } } } class InMemoryStateStore { constructor( private state: StoredState, private readonly unlocked = true ) {} hasUnlockedState(): boolean { return this.unlocked } getStateOrThrow(): StoredState { return this.state } async updateServer( serverId: string, updater: (server: VpsRecord) => VpsRecord ): Promise { const index = this.state.trackedServers.findIndex((server) => server.id === serverId) if (index < 0) { throw new Error("Server not found.") } const next = updater(structuredClone(this.state.trackedServers[index])) this.state.trackedServers[index] = next return structuredClone(next) } async persistNewState(state: StoredState): Promise { this.state = structuredClone(state) } } class TestNodeDeploymentRuntime extends NodeDeploymentRuntime { readonly phases: string[] = [] protected override createPhaseRuntime(): NodeDeploymentPhaseRuntime { return { shouldRunNbdeReconcile: () => true, executeProvisioningProvider: async () => undefined, executeKexec: async (context) => { context.metrics?.mark("kexecStarted") this.phases.push("kexec") return {} as NodeDeploymentInstallSession }, executeInstallOs: async (_context, session) => { this.phases.push("installing-os") return session }, executeRebootInstalledSystem: async (_context, session) => { this.phases.push("rebooting-installed-system") return session }, executeActivateCertificate: async (_context, session) => { this.phases.push("activating-certificate") return session }, executePostInstallTribes: async (_context, session) => { this.phases.push("post-install-tribes") return session }, executeReconcileCluster: async (_context, session) => { this.phases.push("reconciling-cluster") return session }, executeClearPublicationBarrier: async (_context, session) => { this.phases.push("clearing-publication-barrier") return session }, executeWaitReady: async () => { this.phases.push("waiting-ready") return { nodePublicKey: "a".repeat(64), nbde: { luksUuid: "test-luks-uuid" } } }, executeNbdeReconcile: async () => { this.phases.push("nbde-reconciling") } } } } class PhaseRuntimeTestNodeDeploymentRuntime extends NodeDeploymentRuntime { async executeReconcileClusterForTest( context: Parameters[0], session: Parameters[1] ): ReturnType { return this.createPhaseRuntime().executeReconcileCluster(context, session) } } function createNoopPhaseRuntime(): NodeDeploymentPhaseRuntime { return { shouldRunNbdeReconcile: () => false, executeProvisioningProvider: async () => undefined, executeKexec: async () => ({}) as NodeDeploymentInstallSession, executeInstallOs: async (_context, installSession) => installSession, executeRebootInstalledSystem: async (_context, installSession) => installSession, executeActivateCertificate: async (_context, installSession) => installSession, executePostInstallTribes: async (_context, installSession) => installSession, executeReconcileCluster: async (_context, installSession) => installSession, executeClearPublicationBarrier: async (_context, installSession) => installSession, executeWaitReady: async () => ({ nodePublicKey: "a".repeat(64), nbde: {} }), executeNbdeReconcile: async () => undefined } } type RuntimeDecommissionInternals = { createDeploymentService: () => { syncClusterMembership(request: { desiredClusterMembership: StoredState["clusterMembership"] clusterTargets?: ManagedNodeBootstrapState[] preserveExistingMembers?: boolean }): Promise } } type RuntimeClusterInternals = { resolveClusterTargets( currentServerId: string, currentManagedBootstrap: ManagedNodeBootstrapState ): Array<{ server: VpsRecord; managedBootstrap: ManagedNodeBootstrapState }> buildDesiredClusterMembership( overrides?: Map ): StoredState["clusterMembership"] } type RuntimeReconcileInternals = { createDeploymentService: () => { reconcileClusterMembership(session: NodeDeploymentInstallSession): Promise } } type RuntimeNbdeInternals = { createDeploymentService: () => { reconcileNbdeNode(request: { server: VpsRecord nbde: { mode?: string peerTangNodeIds?: string[] peerTangUrls: string[] removeLocalBootKey?: boolean } }): Promise } } test("initialize marks running node executions as interrupted", async () => { const store = new InMemoryStateStore( createState([ createServer({ execution: createExecutionState({ operation: "reinstall", status: "running", phase: "installing-os", input: { kind: "reinstall", install: createInstallInput() } }) }) ]) ) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) await runtime.initialize() const server = store.getStateOrThrow().trackedServers[0] assert.equal(server.execution?.status, "interrupted") assert.equal(server.execution?.retryable, true) assert.equal(server.deployment.status, "failed") }) test("node deployment runtime emits durable/live deployment events", async () => { const store = new InMemoryStateStore(createState([createServer()])) const published: DeploymentActivityEvent[] = [] const recorded: DeploymentActivityEvent[] = [] const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never, publishDeploymentEvent: (event) => published.push(event), recordDeploymentEvent: async (event) => { recorded.push(event) } }) await runtime.reinstallNode("node-a", { kexecImage: "/tmp/guix-kexec-installer.tar.gz", confirmDataLoss: true }) assert.deepEqual( recorded.map((event) => [ event.source, event.nodeId, event.operation, event.phase, event.state ]), [ ["runtime", "node-a", "reinstall", undefined, "started"], ["runtime", "node-a", "reinstall", "kexec", "started"], ["runtime", "node-a", "reinstall", "succeeded", "succeeded"] ] ) assert.deepEqual(published, recorded) }) test("initialize is a no-op while the state store is locked", async () => { const store = new InMemoryStateStore( createState([ createServer({ execution: createExecutionState({ operation: "reinstall", status: "running", phase: "installing-os", input: { kind: "reinstall", install: createInstallInput() } }) }) ]), false ) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) await runtime.initialize() const server = store.getStateOrThrow().trackedServers[0] assert.equal(server.execution?.status, "running") }) test("provisionVps persists bootstrap state", async () => { const store = new InMemoryStateStore(createState([createServer()])) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) await runtime.provisionVps({ id: "node-a" } as never, { provider: "hetzner", providerId: "hetzner" }) const server = store.getStateOrThrow().trackedServers[0] assert.equal(server.managedBootstrap?.publicIpv4, "203.0.113.10") assert.equal(server.execution?.phase, "provisioned") }) test("provisionVps uses product boot mode unless request overrides it", async () => { const store = new InMemoryStateStore(createState([createServer({ bootMode: "efi" })])) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) await runtime.provisionVps({ id: "node-a" } as never, { provider: "hetzner", providerId: "hetzner" }) const catalogInput = store.getStateOrThrow().trackedServers[0].execution?.input assert.equal(catalogInput?.kind, "add") assert.equal(catalogInput.kind === "add" ? catalogInput.install.bootMode : undefined, "efi") await runtime.provisionVps({ id: "node-a" } as never, { provider: "hetzner", providerId: "hetzner", bootMode: "bios" }) const explicitInput = store.getStateOrThrow().trackedServers[0].execution?.input assert.equal(explicitInput?.kind, "add") assert.equal(explicitInput.kind === "add" ? explicitInput.install.bootMode : undefined, "bios") }) test("manual install requests carry the user password for sudo escalation", async () => { const state = createState([ createServer({ providerId: "manual", providerKind: "manual", sshUsername: "debian", managedBootstrap: createManagedBootstrap("1", "node-a", "203.0.113.10") }) ]) state.scheme.servers = [ { id: "node-a", providerKind: "manual", manualConfig: { publicIp: "203.0.113.10", username: "debian", password: "user-secret" } } as never ] const store = new InMemoryStateStore(state) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) as unknown as { buildInstallRequest(plan: unknown): Promise<{ userPassword?: string }> } const request = await runtime.buildInstallRequest({ kind: "add", serverId: "node-a", request: {}, install: createInstallInput() }) assert.equal(request.userPassword, "user-secret") }) test("provisionVps provisions when no tracked server exists", async () => { const store = new InMemoryStateStore(createState([])) let provisionCalls = 0 const runtime = new NodeDeploymentRuntime({ stateStore: store as never, provisionManagedServer: async () => { provisionCalls += 1 const server = createServer() await store.persistNewState({ ...store.getStateOrThrow(), trackedServers: [server] }) return server } }) await runtime.provisionVps({ id: "node-a" } as never, { provider: "hetzner", providerId: "hetzner" }) assert.equal(provisionCalls, 1) assert.equal(store.getStateOrThrow().trackedServers.length, 1) }) test("reinstall and retry drive the shared node deployment actor", async () => { const store = new InMemoryStateStore(createState([createServer()])) const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never }) const reinstallRequest: NodeReinstallRequest = { kexecImage: "/tmp/guix-kexec-installer.tar.gz", confirmDataLoss: true } await runtime.reinstallNode("node-a", reinstallRequest) assert.deepEqual(runtime.phases, [ "kexec", "installing-os", "rebooting-installed-system", "activating-certificate", "post-install-tribes", "waiting-ready", "reconciling-cluster", "clearing-publication-barrier", "nbde-reconciling" ]) assert.equal(store.getStateOrThrow().trackedServers[0].execution?.status, "succeeded") await store.updateServer("node-a", (server) => ({ ...server, execution: createExecutionState({ operation: "reinstall", status: "failed", phase: "failed", input: { kind: "reinstall", install: createInstallInput() }, lastError: "transient failure" }) })) runtime.phases.length = 0 await runtime.retryNode("node-a") assert.deepEqual(runtime.phases, [ "kexec", "installing-os", "rebooting-installed-system", "activating-certificate", "post-install-tribes", "waiting-ready", "reconciling-cluster", "clearing-publication-barrier", "nbde-reconciling" ]) assert.equal(store.getStateOrThrow().trackedServers[0].execution?.status, "succeeded") }) test("reinstall reuses the tracked provider image when it is still in the current catalog", async () => { const store = new InMemoryStateStore( createState( [ createServer({ providerImageId: "ubuntu-24.04", image: "ubuntu-24.04" }) ], [createProvider(true)] ) ) const reinstalledImages: string[] = [] const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never, getProductCatalog: async () => createProductCatalog([ createCatalogProduct("CAX11", "ubuntu-24.04"), createCatalogProduct("CAX21", "ubuntu-26.04") ]), reinstallTrackedServerFromImage: async (serverId, imageId) => { reinstalledImages.push(`${serverId}:${imageId}`) return store.getStateOrThrow().trackedServers[0] } }) await runtime.reinstallNode("node-a", { confirmDataLoss: true }) assert.deepEqual(reinstalledImages, ["node-a:ubuntu-24.04"]) assert.equal(store.getStateOrThrow().trackedServers[0].execution?.operation, "reinstall") }) test("reinstall selects the latest matching catalog image when the tracked provider image is stale", async () => { const store = new InMemoryStateStore( createState( [ createServer({ planName: "CAX11", providerImageId: "ubuntu-22.04", image: "ubuntu-22.04" }) ], [createProvider(true)] ) ) const reinstalledImages: string[] = [] const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never, getProductCatalog: async () => createProductCatalog([ createCatalogProduct("CAX11", "ubuntu-26.04"), createCatalogProduct("CAX21", "ubuntu-26.04-large") ]), reinstallTrackedServerFromImage: async (serverId, imageId) => { reinstalledImages.push(`${serverId}:${imageId}`) return store.getStateOrThrow().trackedServers[0] } }) await runtime.reinstallNode("node-a", { confirmDataLoss: true }) assert.deepEqual(reinstalledImages, ["node-a:ubuntu-26.04"]) }) test("reinstall skips provider image reset when the provider does not support it", async () => { const store = new InMemoryStateStore(createState([createServer()], [createProvider(false)])) const reinstalledImages: string[] = [] const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never, getProductCatalog: async () => createProductCatalog([createCatalogProduct("CAX11", "ubuntu-26.04")]), reinstallTrackedServerFromImage: async (serverId, imageId) => { reinstalledImages.push(`${serverId}:${imageId}`) return store.getStateOrThrow().trackedServers[0] } }) await runtime.reinstallNode("node-a", { confirmDataLoss: true }) assert.deepEqual(reinstalledImages, []) assert.equal(store.getStateOrThrow().trackedServers[0].execution?.operation, "reinstall") }) test("reinstall requires confirmation when it would reinitialize the only managed cluster node", async () => { const store = new InMemoryStateStore( createState([ createServer({ managedBootstrap: createManagedBootstrap("1", "node-a", "203.0.113.10") }) ]) ) const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never }) await assert.rejects(runtime.reinstallNode("node-a"), /Confirm data loss to continue/) }) test("reinstall requires confirmation for an unhealthy managed node when no healthy peer exists", async () => { const store = new InMemoryStateStore( createState([ createServer({ status: "error", deployment: { status: "failed", snippets: [] }, managedBootstrap: createManagedBootstrap("1", "node-a", "203.0.113.10") }) ]) ) const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never }) await assert.rejects(runtime.reinstallNode("node-a"), /Confirm data loss to continue/) }) test("reinstall does not require data-loss confirmation when another healthy node exists", async () => { const store = new InMemoryStateStore( createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: createManagedBootstrap("1", "node-a", "203.0.113.10") }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: createManagedBootstrap("2", "node-b", "203.0.113.11") }) ]) ) const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never }) await runtime.reinstallNode("node-a") assert.equal(store.getStateOrThrow().trackedServers[0].execution?.operation, "reinstall") }) test("node deployment actor exposes provisioning metrics on machine context", async () => { await writeFile("/tmp/guix-kexec-installer.tar.gz", "test", "utf8") const store = new InMemoryStateStore(createState([createServer()])) const marks: string[] = [] const finishes: string[] = [] const operations: string[] = [] const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never, provisioningMetrics: { startRun(input) { operations.push(input.operation) return { mark(milestone) { marks.push(milestone) }, mergeMetadata() { return undefined }, async finish(input) { finishes.push(input.result) } } } } }) await runtime.reinstallNode("node-a", { kexecImage: "/tmp/guix-kexec-installer.tar.gz", confirmDataLoss: true }) assert.deepEqual(operations, ["reinstall"]) assert.ok(marks.includes("kexecStarted")) assert.deepEqual(finishes, ["succeeded"]) }) test("retry resumes add from a later persisted phase without going back to kexec", async () => { await writeFile("/tmp/guix-kexec-installer.tar.gz", "test", "utf8") const store = new InMemoryStateStore( createState([ createServer({ sshPublicKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITest legion@test", managedBootstrap: { nodePrivateKey: "1".repeat(64), nodeName: "node-a", publicIpv4: "203.0.113.10", publicPort: 443, publicScheme: "https" }, execution: createExecutionState({ operation: "add", status: "failed", phase: "reconciling-cluster", input: { kind: "add", install: createInstallInput() }, lastError: "upstream unavailable" }) }) ]) ) const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never }) await runtime.retryNode("node-a") assert.deepEqual(runtime.phases, [ "reconciling-cluster", "clearing-publication-barrier", "nbde-reconciling" ]) assert.equal(store.getStateOrThrow().trackedServers[0].execution?.status, "succeeded") }) test("bulk NBDE reconcile updates execution state on every target node", async () => { const store = new InMemoryStateStore( createState([ createServer({ id: "node-a" }), createServer({ id: "node-b", publicIp: "203.0.113.11" }) ]) ) const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never }) await runtime.reconcileNbde(["node-a", "node-b"]) assert.equal(store.getStateOrThrow().trackedServers[0].execution?.operation, "nbde-reconcile") assert.equal(store.getStateOrThrow().trackedServers[0].execution?.status, "succeeded") assert.equal(store.getStateOrThrow().trackedServers[1].execution?.operation, "nbde-reconcile") assert.equal(store.getStateOrThrow().trackedServers[1].execution?.status, "succeeded") }) test("bulk NBDE reconcile derives clustered policy from a stable target snapshot", async () => { const store = new InMemoryStateStore( createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: createManagedBootstrap("1", "node-a", "203.0.113.10") }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: createManagedBootstrap("2", "node-b", "203.0.113.11") }), createServer({ id: "node-c", publicIp: "203.0.113.12", managedBootstrap: createManagedBootstrap("3", "node-c", "203.0.113.12") }) ]) ) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) const requests: Array<{ serverId: string; mode?: string; peerTangNodeIds?: string[] }> = [] const runtimeInternals = runtime as unknown as RuntimeNbdeInternals runtimeInternals.createDeploymentService = () => ({ reconcileNbdeNode: async (request) => { requests.push({ serverId: request.server.id, mode: request.nbde.mode, peerTangNodeIds: request.nbde.peerTangNodeIds }) } }) await runtime.reconcileNbde(["node-a", "node-b", "node-c"]) assert.deepEqual( requests.map((request) => [request.serverId, request.mode, request.peerTangNodeIds?.sort()]), [ ["node-a", "clustered", ["node-b", "node-c"]], ["node-b", "clustered", ["node-a", "node-c"]], ["node-c", "clustered", ["node-a", "node-b"]] ] ) assert.deepEqual( store .getStateOrThrow() .trackedServers.map((server) => [ server.id, server.nbde?.mode, server.nbde?.peerTangNodeIds.sort(), server.nbde?.localBootKeyPresent ]), [ ["node-a", "clustered", ["node-b", "node-c"], false], ["node-b", "clustered", ["node-a", "node-c"], false], ["node-c", "clustered", ["node-a", "node-b"], false] ] ) }) test("clustered NBDE reconcile removes temporary local boot keys", async () => { const primaryBootstrap = createManagedBootstrap("1", "node-a", "203.0.113.10") const joinBootstrap = createManagedBootstrap("2", "node-b", "203.0.113.11") const store = new InMemoryStateStore( createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: primaryBootstrap }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: joinBootstrap, nbde: { mode: "clustered", tangPort: 7654, recoverySecret: "test-recovery-secret", localBootKeyPresent: true, peerTangNodeIds: ["node-a"], installedAt: "2026-04-06T12:00:00.000Z" } }), createServer({ id: "node-c", publicIp: "203.0.113.12", managedBootstrap: createManagedBootstrap("3", "node-c", "203.0.113.12") }) ]) ) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) let removeLocalBootKey: boolean | undefined const runtimeInternals = runtime as unknown as RuntimeNbdeInternals runtimeInternals.createDeploymentService = () => ({ reconcileNbdeNode: async (request) => { removeLocalBootKey = request.nbde.removeLocalBootKey } }) await runtime.reconcileNbde(["node-b"]) assert.equal(removeLocalBootKey, true) assert.equal(store.getStateOrThrow().trackedServers[1]?.nbde?.localBootKeyPresent, false) }) test("reconfigureManagedNodes runs the post-install Tribes phases without kexec", async () => { const store = new InMemoryStateStore( createState([ createServer({ sshPublicKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITest legion@test", managedBootstrap: { nodePrivateKey: "1".repeat(64), nodeName: "node-a", publicIpv4: "203.0.113.10", publicPort: 443, publicScheme: "https" } }) ]) ) const runtime = new TestNodeDeploymentRuntime({ stateStore: store as never }) await runtime.reconfigureManagedNodes(["node-a"]) assert.deepEqual(runtime.phases, [ "post-install-tribes", "waiting-ready", "reconciling-cluster", "clearing-publication-barrier", "nbde-reconciling" ]) assert.equal(store.getStateOrThrow().trackedServers[0].execution?.operation, "tribes-reconfigure") assert.equal(store.getStateOrThrow().trackedServers[0].execution?.status, "succeeded") }) test("decommissionManagedNode syncs survivors and preserves decommissioned membership rows", async () => { const state = createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: { nodePrivateKey: "1".repeat(64), nodeName: "node-a", publicIpv4: "203.0.113.10", publicPort: 443, publicScheme: "https", installedAt: "2026-04-06T12:00:00.000Z" } }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: { nodePrivateKey: "2".repeat(64), nodeName: "node-b", publicIpv4: "203.0.113.11", publicPort: 443, publicScheme: "https", installedAt: "2026-04-07T12:00:00.000Z" } }), createServer({ id: "node-d", publicIp: "203.0.113.13", deployment: createDeploymentState("failed"), execution: createExecutionState({ operation: "add", phase: "waiting-ready", status: "failed" }), managedBootstrap: { nodePrivateKey: "4".repeat(64), nodeName: "node-d", publicIpv4: "203.0.113.13", publicPort: 443, publicScheme: "https", installedAt: "2026-04-07T12:05:00.000Z" } }) ]) state.tribe = { name: "Demo Tribe", domain: "example.com", password: "bootstrap-secret" } state.clusterBootstrap = createTestClusterBootstrap() state.clusterMembership = [ { nodeId: "node-a", pubkey: deriveNodePublicKey("1".repeat(64)), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" }, { nodeId: "node-c", pubkey: "c".repeat(64), transportAddress: "wss://203.0.113.12:4413/relay", scope: "all", status: "decommissioned", activatedAt: "2026-04-05T12:00:00.000Z", deactivatedAt: "2026-04-08T12:00:00.000Z" } ] const store = new InMemoryStateStore(state) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) let syncedMembership: StoredState["clusterMembership"] | null = null let syncedClusterTargets: ManagedNodeBootstrapState[] | undefined let syncedPreserveExistingMembers: boolean | undefined const runtimeInternals = runtime as unknown as RuntimeDecommissionInternals runtimeInternals.createDeploymentService = () => ({ syncClusterMembership: async (request: { desiredClusterMembership: StoredState["clusterMembership"] clusterTargets?: ManagedNodeBootstrapState[] preserveExistingMembers?: boolean }) => { syncedMembership = structuredClone(request.desiredClusterMembership) syncedClusterTargets = structuredClone(request.clusterTargets) syncedPreserveExistingMembers = request.preserveExistingMembers } }) await runtime.decommissionManagedNode("node-b") assert.deepEqual(syncedMembership, [ { nodeId: "node-b", pubkey: deriveNodePublicKey("2".repeat(64)), transportAddress: "wss://203.0.113.11:4413/relay", scope: "all", status: "decommissioned", activatedAt: "2026-04-07T12:00:00.000Z", deactivatedAt: store .getStateOrThrow() .clusterMembership.find((entry) => entry.nodeId === "node-b")?.deactivatedAt }, { nodeId: "node-a", pubkey: deriveNodePublicKey("1".repeat(64)), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z", deactivatedAt: undefined }, { nodeId: "node-c", pubkey: "c".repeat(64), transportAddress: "wss://203.0.113.12:4413/relay", scope: "all", status: "decommissioned", activatedAt: "2026-04-05T12:00:00.000Z", deactivatedAt: "2026-04-08T12:00:00.000Z" } ]) assert.deepEqual( syncedClusterTargets?.map((target) => target.publicIpv4), ["203.0.113.10"] ) assert.equal(syncedPreserveExistingMembers, true) assert.deepEqual(store.getStateOrThrow().clusterMembership, syncedMembership) }) test("decommissionManagedNode clears local cluster membership when removing the last clustered node", async () => { const state = createState([ createServer({ managedBootstrap: { nodePrivateKey: "1".repeat(64), nodeName: "node-a", publicIpv4: "203.0.113.10", publicPort: 443, publicScheme: "https" } }) ]) state.tribe = { name: "Demo Tribe", domain: "example.com", password: "bootstrap-secret" } state.clusterBootstrap = createTestClusterBootstrap() state.clusterMembership = [ { nodeId: "node-a", pubkey: deriveNodePublicKey("1".repeat(64)), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" } ] const store = new InMemoryStateStore(state) const runtime = new NodeDeploymentRuntime({ stateStore: store as never }) let syncCalls = 0 const runtimeInternals = runtime as unknown as RuntimeDecommissionInternals runtimeInternals.createDeploymentService = () => ({ syncClusterMembership: async () => { syncCalls += 1 } }) await runtime.decommissionManagedNode("node-a") assert.equal(syncCalls, 0) assert.deepEqual(store.getStateOrThrow().clusterMembership, []) }) test("cluster install requests ignore planned peers until they have joined", () => { const primaryBootstrap = createManagedBootstrap("1", "node-a", "203.0.113.10") const joinBootstrap = createManagedBootstrap("2", "node-b", "203.0.113.11") const plannedPeerBootstrap = createManagedBootstrap("3", "node-c", "203.0.113.12") const state = createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: primaryBootstrap }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: joinBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "waiting-ready", input: { kind: "add", install: createInstallInput() } }) }), createServer({ id: "node-c", publicIp: "203.0.113.12", managedBootstrap: plannedPeerBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "provisioned", input: { kind: "add", install: createInstallInput() } }) }) ]) state.clusterMembership = [ { nodeId: "node-a", pubkey: deriveNodePublicKey(primaryBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" } ] const runtime = new NodeDeploymentRuntime({ stateStore: new InMemoryStateStore(state) as never }) as unknown as RuntimeClusterInternals const clusterTargets = runtime.resolveClusterTargets("node-b", joinBootstrap) assert.deepEqual( clusterTargets.map((entry) => entry.server.id), ["node-a", "node-b"] ) const desiredMembership = runtime.buildDesiredClusterMembership( new Map([["node-b", joinBootstrap]]) ) assert.deepEqual(desiredMembership.map((entry) => entry.nodeId).sort(), ["node-a", "node-b"]) }) test("cluster membership includes a planned peer after it becomes the active install target", () => { const primaryBootstrap = createManagedBootstrap("1", "node-a", "203.0.113.10") const joinedBootstrap = createManagedBootstrap("2", "node-b", "203.0.113.11") const currentBootstrap = createManagedBootstrap("3", "node-c", "203.0.113.12") const state = createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: primaryBootstrap }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: joinedBootstrap }), createServer({ id: "node-c", publicIp: "203.0.113.12", managedBootstrap: currentBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "waiting-ready", input: { kind: "add", install: createInstallInput() } }) }) ]) state.clusterMembership = [ { nodeId: "node-a", pubkey: deriveNodePublicKey(primaryBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" }, { nodeId: "node-b", pubkey: deriveNodePublicKey(joinedBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.11:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-07T12:00:00.000Z" } ] const runtime = new NodeDeploymentRuntime({ stateStore: new InMemoryStateStore(state) as never }) as unknown as RuntimeClusterInternals const clusterTargets = runtime.resolveClusterTargets("node-c", currentBootstrap) assert.deepEqual( clusterTargets.map((entry) => entry.server.id), ["node-a", "node-b", "node-c"] ) const desiredMembership = runtime.buildDesiredClusterMembership( new Map([["node-c", currentBootstrap]]) ) assert.deepEqual(desiredMembership.map((entry) => entry.nodeId).sort(), [ "node-a", "node-b", "node-c" ]) }) test("cluster membership includes a parallel peer once it reaches bootstrap wait", () => { const primaryBootstrap = createManagedBootstrap("1", "node-a", "203.0.113.10") const parallelBootstrap = createManagedBootstrap("2", "node-b", "203.0.113.11") const currentBootstrap = createManagedBootstrap("3", "node-c", "203.0.113.12") const state = createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: primaryBootstrap }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: parallelBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "waiting-ready", input: { kind: "add", install: createInstallInput() } }) }), createServer({ id: "node-c", publicIp: "203.0.113.12", managedBootstrap: currentBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "reconciling-cluster", input: { kind: "add", install: createInstallInput() } }) }) ]) state.clusterMembership = [ { nodeId: "node-a", pubkey: deriveNodePublicKey(primaryBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" } ] const runtime = new NodeDeploymentRuntime({ stateStore: new InMemoryStateStore(state) as never }) as unknown as RuntimeClusterInternals const clusterTargets = runtime.resolveClusterTargets("node-c", currentBootstrap) assert.deepEqual( clusterTargets.map((entry) => entry.server.id), ["node-a", "node-b", "node-c"] ) const desiredMembership = runtime.buildDesiredClusterMembership( new Map([["node-c", currentBootstrap]]) ) assert.deepEqual(desiredMembership.map((entry) => entry.nodeId).sort(), [ "node-a", "node-b", "node-c" ]) }) test("reconcile cluster phase recomputes membership after parallel peers finish", async () => { const primaryBootstrap = createManagedBootstrap("1", "node-a", "203.0.113.10") const joinedBootstrap = createManagedBootstrap("2", "node-b", "203.0.113.11") const currentBootstrap = createManagedBootstrap("3", "node-c", "203.0.113.12") const state = createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: primaryBootstrap }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: joinedBootstrap, execution: createExecutionState({ operation: "add", status: "succeeded", phase: "succeeded", input: { kind: "add", install: createInstallInput() } }) }), createServer({ id: "node-c", publicIp: "203.0.113.12", managedBootstrap: currentBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "waiting-ready", input: { kind: "add", install: createInstallInput() } }) }) ]) state.clusterMembership = [ { nodeId: "node-a", pubkey: deriveNodePublicKey(primaryBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" }, { nodeId: "node-b", pubkey: deriveNodePublicKey(joinedBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.11:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-07T12:00:00.000Z" } ] const store = new InMemoryStateStore(state) const runtime = new PhaseRuntimeTestNodeDeploymentRuntime({ stateStore: store as never }) const reconcileCapture: { request?: NodeDeploymentInstallSession["request"] } = {} const runtimeInternals = runtime as unknown as RuntimeReconcileInternals runtimeInternals.createDeploymentService = () => ({ reconcileClusterMembership: async (session: NodeDeploymentInstallSession) => { reconcileCapture.request = structuredClone(session.request) } }) await runtime.executeReconcileClusterForTest( { operationId: "op-add-c", phaseRuntime: createNoopPhaseRuntime(), plan: { kind: "add", serverId: "node-c", request: { provider: "hetzner", providerId: "provider-1", offer: "default" }, install: createInstallInput() } } as never, { request: { server: state.trackedServers[2]!, managedBootstrap: currentBootstrap, desiredClusterMembership: [ { nodeId: "node-a", pubkey: deriveNodePublicKey(primaryBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" }, { nodeId: "node-c", pubkey: deriveNodePublicKey(currentBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.12:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" } ], clusterTargets: [ { server: state.trackedServers[0]!, managedBootstrap: primaryBootstrap }, { server: state.trackedServers[2]!, managedBootstrap: currentBootstrap } ], nbde: { mode: "degraded", tangPort: 7654, recoverySecret: "test-recovery-secret", localBootKeyPresent: true, peerTangNodeIds: [], installedAt: "2026-04-06T12:00:00.000Z", peerTangUrls: [] }, acmeEmail: "hostmaster@example.com", bootMode: "efi" } } as unknown as NodeDeploymentInstallSession ) assert.ok(reconcileCapture.request) assert.deepEqual( reconcileCapture.request.clusterTargets.map((entry) => entry.server.id), ["node-a", "node-b", "node-c"] ) assert.deepEqual( reconcileCapture.request.desiredClusterMembership.map((entry) => entry.nodeId).sort(), ["node-a", "node-b", "node-c"] ) assert.deepEqual( store .getStateOrThrow() .clusterMembership.map((entry) => entry.nodeId) .sort(), ["node-a", "node-b", "node-c"] ) }) test("reconcile cluster phase persists peers that become ready while service reconciles", async () => { const primaryBootstrap = createManagedBootstrap("1", "node-a", "203.0.113.10") const parallelBootstrap = createManagedBootstrap("2", "node-b", "203.0.113.11") const currentBootstrap = createManagedBootstrap("3", "node-c", "203.0.113.12") const state = createState([ createServer({ id: "node-a", publicIp: "203.0.113.10", managedBootstrap: primaryBootstrap }), createServer({ id: "node-b", publicIp: "203.0.113.11", managedBootstrap: parallelBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "post-install-tribes", input: { kind: "add", install: createInstallInput() } }) }), createServer({ id: "node-c", publicIp: "203.0.113.12", managedBootstrap: currentBootstrap, deployment: createDeploymentState("running"), execution: createExecutionState({ operation: "add", status: "running", phase: "waiting-ready", input: { kind: "add", install: createInstallInput() } }) }) ]) state.clusterMembership = [ { nodeId: "node-a", pubkey: deriveNodePublicKey(primaryBootstrap.nodePrivateKey), transportAddress: "wss://203.0.113.10:4413/relay", scope: "all", status: "active", activatedAt: "2026-04-06T12:00:00.000Z" } ] const store = new InMemoryStateStore(state) const runtime = new PhaseRuntimeTestNodeDeploymentRuntime({ stateStore: store as never }) const reconcileCapture: { request?: NodeDeploymentInstallSession["request"] } = {} const runtimeInternals = runtime as unknown as RuntimeReconcileInternals runtimeInternals.createDeploymentService = () => ({ reconcileClusterMembership: async (session: NodeDeploymentInstallSession) => { reconcileCapture.request = structuredClone(session.request) await store.updateServer("node-b", (server) => ({ ...server, execution: server.execution ? { ...server.execution, phase: "waiting-ready" } : server.execution })) } }) await runtime.executeReconcileClusterForTest( { operationId: "op-add-c", phaseRuntime: createNoopPhaseRuntime(), plan: { kind: "add", serverId: "node-c", request: { provider: "hetzner", providerId: "provider-1", offer: "default" }, install: createInstallInput() } } as never, { request: { server: state.trackedServers[2]!, managedBootstrap: currentBootstrap, desiredClusterMembership: [], clusterTargets: [], nbde: { mode: "degraded", tangPort: 7654, recoverySecret: "test-recovery-secret", localBootKeyPresent: true, peerTangNodeIds: [], installedAt: "2026-04-06T12:00:00.000Z", peerTangUrls: [] }, acmeEmail: "hostmaster@example.com", bootMode: "efi" } } as unknown as NodeDeploymentInstallSession ) assert.ok(reconcileCapture.request) assert.deepEqual( reconcileCapture.request.desiredClusterMembership.map((entry) => entry.nodeId).sort(), ["node-a", "node-c"] ) assert.deepEqual( store .getStateOrThrow() .clusterMembership.map((entry) => entry.nodeId) .sort(), ["node-a", "node-b", "node-c"] ) }) test("provider deployment profiles define boot mode defaults", () => { assert.equal(getProviderDeploymentProfile("hetzner").defaultBootMode, "auto") assert.equal(getProviderDeploymentProfile("ovh").defaultBootMode, "auto") assert.equal(getProviderDeploymentProfile("scaleway").defaultBootMode, "auto") }) function createInstallInput(): NodeExecutionInstallInput { return { kexecImagePath: "/tmp/guix-kexec-installer.tar.gz", bootMode: "efi" } } function createExecutionState( overrides: Partial & Pick ): NodeExecutionState { return { operationId: "op-1", operation: overrides.operation, status: overrides.status, phase: overrides.phase, retryable: overrides.retryable ?? true, startedAt: "2026-04-06T12:00:00.000Z", updatedAt: "2026-04-06T12:00:00.000Z", finishedAt: overrides.finishedAt, lastError: overrides.lastError, input: overrides.input } } function createManagedBootstrap( keySeed: string, nodeName: string, publicIpv4: string ): ManagedNodeBootstrapState { return { nodePrivateKey: keySeed.repeat(64), nodeName, publicIpv4, publicPort: 443, publicScheme: "https", installedAt: "2026-04-06T12:00:00.000Z" } } function createDeploymentState(status: VpsRecord["deployment"]["status"]): VpsRecord["deployment"] { return { status, snippets: [] } } function createProvider(computeReinstallFromImage: boolean): ProviderConfig { return { id: "hetzner", kind: "hetzner", credentials: { kind: "hetzner", apiKey: "token" }, capabilities: { computeCatalog: true, computeProvisioning: true, computeReinstallFromImage, domainRegistration: false, standardDnsService: true, fallbackDnsService: false, dnsRecordManagement: true, firewallManagement: true }, configured: true, createdAt: "2026-04-06T12:00:00.000Z" } } function createProductCatalog(products: ProductCatalog["products"]): ProductCatalog { return { providerKind: "hetzner", isDefaultProvider: true, products, generatedAt: "2026-04-06T12:00:00.000Z", disclaimer: "" } } function createCatalogProduct(planName: string, image: string): ProductCatalog["products"][number] { return { id: `hetzner:${planName}:${image}`, providerKind: "hetzner", providerProductId: planName, label: planName, planName, description: planName, architecture: "x86_64", category: "shared", cores: 2, cpuType: "shared", memoryGb: 4, diskGb: 40, storageType: "ssd", image, bootMode: "auto", defaultRegion: "fsn1", availableRegions: ["fsn1"], defaultPrice: { region: "fsn1", hourlyGrossEur: 0.006, hourlyNetEur: 0.005, monthlyGrossEur: 4, monthlyNetEur: 3.36 }, prices: [ { region: "fsn1", hourlyGrossEur: 0.006, hourlyNetEur: 0.005, monthlyGrossEur: 4, monthlyNetEur: 3.36 } ], commercialMetadata: { effectiveMonthlyGrossEur: 4, source: "provider", notes: [] }, recommended: false } } function createState(trackedServers: VpsRecord[], providers: ProviderConfig[] = []): StoredState { return { version: APP_STATE_VERSION, tribe: { name: "Test Tribe", domain: "example.test", password: "test-tribe-password" }, clusterBootstrap: createTestClusterBootstrap(), clusterMembership: [], profile: null, providers, bindings: [], pendingProvisioning: [], trackedServers, scheme: { servers: [], domains: [], dnsHosts: [], dnsZones: [] }, actual: { servers: [], domains: [], dnsZones: [] }, tutorial: { completed: false } } } function createServer(overrides: Partial = {}): VpsRecord { return { id: "node-a", providerId: "hetzner", providerKind: "hetzner", role: "primary", sshUsername: "root", sshPrivateKey: "private-key", status: "running", publicIp: "203.0.113.10", region: "fsn1", planName: "CAX11", commercialMetadata: { effectiveMonthlyGrossEur: 4, source: "provider", notes: [] }, deployment: { status: "ready", snippets: [] }, nbde: { mode: "degraded", tangPort: 7654, recoverySecret: "test-recovery-secret", localBootKeyPresent: true, peerTangNodeIds: [], installedAt: "2026-04-06T12:00:00.000Z" }, createdAt: "2026-04-06T12:00:00.000Z", updatedAt: "2026-04-06T12:00:00.000Z", ...overrides } }