self
cfbed53f96
Use ProviderId as the canonical provider model identity and remove resource-level/provider-kind/source model duplication. Treat manual as the provider id for manual servers and external domains, deriving domain DNS association from same-name zones.
1772 lines
56 KiB
TypeScript
1772 lines
56 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { mkdtemp, rm, writeFile } from "node:fs/promises"
|
|
import { tmpdir } from "node:os"
|
|
import { join } from "node:path"
|
|
import test from "node:test"
|
|
|
|
import {
|
|
NodeDeploymentService,
|
|
parseInternalStatusNodePublicKey,
|
|
parseBootstrapProbeState,
|
|
parseLegionDeploymentStatusMarker,
|
|
summarizeAcmeCertificateIssue,
|
|
type ClusterMembershipSyncRequest,
|
|
type NodeDeploymentReporter,
|
|
type NodeDeploymentInstallSession
|
|
} from "../../src/main/deployment/service"
|
|
import { deriveNodePublicKey, encryptNostrPrivateKey } from "../../src/main/bootstrap-crypto"
|
|
import type {
|
|
ClusterBootstrapState,
|
|
ClusterMembershipRecord,
|
|
LegionAdminConfig,
|
|
VpsRecord
|
|
} from "../../src/shared/app"
|
|
import type { TribesDeploymentProfile, TribesHostConfig } from "../../src/shared/tribes-deployment"
|
|
|
|
const testDeploymentProfile: TribesDeploymentProfile = {
|
|
schemaVersion: "1",
|
|
plugins: [],
|
|
tribes: {
|
|
workingDirectory: "/var/lib/tribes",
|
|
serviceUser: "tribes",
|
|
serviceGroup: "tribes",
|
|
listenAddress: "127.0.0.1",
|
|
listenPort: 4000,
|
|
publicScheme: "https",
|
|
publicPort: 443,
|
|
syncPort: 4413,
|
|
syncBindAddress: "0.0.0.0",
|
|
syncOverlapSeconds: 300,
|
|
databaseUser: "tribes",
|
|
databaseName: "tribes",
|
|
parrhesiaDatabaseName: "parrhesia",
|
|
databaseHost: "/var/run/postgresql",
|
|
adminPubkeys: [],
|
|
releaseDistribution: "none",
|
|
logFile: "/var/log/tribes.log"
|
|
},
|
|
edge: {
|
|
certificateProfile: "shortlived",
|
|
renewDays: 4,
|
|
httpPort: 80,
|
|
httpsPort: 443,
|
|
challengeAddress: "127.0.0.1",
|
|
challengePort: 8080,
|
|
cacheAddress: "127.0.0.1",
|
|
cachePort: 6081,
|
|
cacheStorage: ["malloc,256M"]
|
|
}
|
|
}
|
|
|
|
const testHostConfig: TribesHostConfig = {
|
|
schemaVersion: "1",
|
|
tribes: {
|
|
workingDirectory: "/var/lib/tribes",
|
|
serviceUser: "tribes",
|
|
serviceGroup: "tribes",
|
|
host: "203.0.113.10",
|
|
listenAddress: "127.0.0.1",
|
|
listenPort: 4000,
|
|
scheme: "https",
|
|
port: 443,
|
|
syncHost: "203.0.113.10",
|
|
syncPort: 4413,
|
|
syncBindAddress: "0.0.0.0",
|
|
adminPubkeys: [],
|
|
plugins: [],
|
|
syncOverlapSeconds: 300,
|
|
databaseUser: "tribes",
|
|
databaseName: "tribes",
|
|
parrhesiaDatabaseName: "parrhesia",
|
|
databaseHost: "/var/run/postgresql",
|
|
secretKeyBaseFile: "/var/lib/tribes/secrets/secret_key_base",
|
|
tokenSigningSecretFile: "/var/lib/tribes/secrets/token_signing_secret",
|
|
releaseCookieFile: "/var/lib/tribes/secrets/release_cookie",
|
|
releaseDistribution: "none",
|
|
extraEnvironmentVariables: ["TRIBES_BOOTSTRAP_FILE=/etc/tribes/bootstrap.json"],
|
|
logFile: "/var/log/tribes.log"
|
|
},
|
|
edge: {
|
|
certificateName: "203-0-113-10",
|
|
certificateSubjects: ["203.0.113.10"],
|
|
certificateProfile: "shortlived",
|
|
renewDays: 4,
|
|
httpPort: 80,
|
|
httpsPort: 443,
|
|
challengeAddress: "127.0.0.1",
|
|
challengePort: 8080,
|
|
cacheAddress: "127.0.0.1",
|
|
cachePort: 6081,
|
|
cacheStorage: ["malloc,256M"]
|
|
}
|
|
}
|
|
|
|
type ServiceTestInternals = {
|
|
createAdminApiClient: (
|
|
request: { clusterBootstrap?: unknown },
|
|
target: { publicIpv4?: string }
|
|
) => {
|
|
clusterNodeUpsert(input: {
|
|
pubkey: string
|
|
transportAddress?: string
|
|
scope?: ClusterMembershipRecord["scope"]
|
|
status?: ClusterMembershipRecord["status"]
|
|
activatedAt?: string
|
|
deactivatedAt?: string
|
|
}): Promise<void>
|
|
clusterNodesList?(): Promise<{
|
|
nodes: Array<{
|
|
pubkey: string
|
|
transport_address: string
|
|
scope: ClusterMembershipRecord["scope"]
|
|
status: ClusterMembershipRecord["status"]
|
|
activated_at?: string
|
|
deactivated_at?: string | null
|
|
}>
|
|
}>
|
|
clusterStatus?(): Promise<{
|
|
servers: Array<{ auth_pubkey: string }>
|
|
}>
|
|
}
|
|
readClusterMembershipViews: (
|
|
request: { clusterBootstrap?: unknown },
|
|
targets: Array<{ publicIpv4?: string }>
|
|
) => Promise<Array<{ host: string; nodes: ClusterMembershipRecord[] }>>
|
|
}
|
|
|
|
function createTestClusterBootstrap(): ClusterBootstrapState {
|
|
return {
|
|
rootSecret: "root-secret",
|
|
relayPrivateKey: "relay-private-key",
|
|
legionAdmin: {
|
|
username: "legion-admin",
|
|
passwordHash: "password-hash",
|
|
nostrPublicKey: "f".repeat(64),
|
|
encryptedNostrPrivateKey: "ciphertext",
|
|
nostrPrivateKeyNonce: "nonce",
|
|
nostrPrivateKeySalt: "salt"
|
|
}
|
|
}
|
|
}
|
|
|
|
function createTestLegionAdmin(): LegionAdminConfig {
|
|
return {
|
|
id: "legion-managed-admin",
|
|
username: "legion-admin",
|
|
role: "admin",
|
|
publicKeyHex: "f".repeat(64),
|
|
npub: "npub-test",
|
|
encryptedPrivateKey: encryptNostrPrivateKey("root-secret", "1".repeat(64)),
|
|
createdAt: "2026-01-01T00:00:00.000Z",
|
|
updatedAt: "2026-01-01T00:00:00.000Z"
|
|
}
|
|
}
|
|
|
|
function createManagedBootstrapSession(pubkey: string): NodeDeploymentInstallSession {
|
|
return {
|
|
request: {
|
|
server: {
|
|
id: "node-a",
|
|
publicIp: "203.0.113.10",
|
|
sshUsername: "root",
|
|
sshPrivateKey: "private-key"
|
|
},
|
|
bootstrapMode: "init",
|
|
managedBootstrap: {
|
|
nodePrivateKey: "a".repeat(64),
|
|
nodePublicKey: pubkey,
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
},
|
|
nbde: {
|
|
mode: "degraded",
|
|
tangPort: 7654,
|
|
recoverySecret: "secret",
|
|
localBootKeyPresent: true,
|
|
peerTangNodeIds: [],
|
|
peerTangUrls: []
|
|
},
|
|
kexecImagePath: "/tmp/guix-kexec-installer.tar.gz",
|
|
bootMode: "bios"
|
|
} as unknown as NodeDeploymentInstallSession["request"],
|
|
ssh: {
|
|
host: "203.0.113.10",
|
|
username: "root",
|
|
privateKey: "private-key"
|
|
},
|
|
nodePublicKey: pubkey,
|
|
deploymentProfile: testDeploymentProfile,
|
|
hostConfig: testHostConfig,
|
|
hostConfigJson: `${JSON.stringify(testHostConfig, null, 2)}\n`,
|
|
bootstrapJson: '{\n "contract_version": "1"\n}\n',
|
|
serviceSecrets: {
|
|
secretKeyBase: "secret-key-base",
|
|
tokenSigningSecret: "token-signing-secret"
|
|
},
|
|
releaseCookie: "release-cookie",
|
|
publicHost: "203.0.113.10"
|
|
}
|
|
}
|
|
|
|
async function immediateDelay<T = void>(_delay?: number, value?: T): Promise<T> {
|
|
return value as T
|
|
}
|
|
|
|
function createClusterMembershipTarget(
|
|
seed: string,
|
|
host: string
|
|
): {
|
|
nodePrivateKey: string
|
|
publicIpv4: string
|
|
publicPort: number
|
|
publicScheme: "https"
|
|
} {
|
|
return {
|
|
nodePrivateKey: seed.repeat(64),
|
|
publicIpv4: host,
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
}
|
|
}
|
|
|
|
function createClusterMembershipRecord(seed: string, host: string): ClusterMembershipRecord {
|
|
return {
|
|
pubkey: seed.repeat(64),
|
|
transportAddress: `wss://${host}:4413/relay`,
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-06T12:00:00.000Z"
|
|
}
|
|
}
|
|
|
|
function createServerRecord(overrides: Partial<VpsRecord> = {}): VpsRecord {
|
|
return {
|
|
id: "node-a",
|
|
provider: "hetzner",
|
|
role: "primary",
|
|
sshUsername: "root",
|
|
sshPrivateKey: "private-key",
|
|
status: "running",
|
|
publicIp: "203.0.113.10",
|
|
region: "fsn1",
|
|
planName: "CAX11",
|
|
commercialMetadata: {
|
|
effectiveMonthlyGrossEur: 4,
|
|
source: "provider",
|
|
notes: []
|
|
},
|
|
deployment: {
|
|
status: "ready",
|
|
snippets: []
|
|
},
|
|
createdAt: "2026-04-06T12:00:00.000Z",
|
|
updatedAt: "2026-04-06T12:00:00.000Z",
|
|
...overrides
|
|
}
|
|
}
|
|
|
|
test("reconcileNbdeNode uses transient SSH retry wrappers", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
let uploaded = false
|
|
let command: string | undefined
|
|
|
|
service["uploadSshFilesWithTransientSshRetry"] = async (
|
|
ssh: { host: string },
|
|
files: Array<{ remotePath: string; mode: number }>
|
|
) => {
|
|
uploaded = true
|
|
assert.equal(ssh.host, "203.0.113.10")
|
|
assert.equal(files[0]?.remotePath, "/tmp/legion/nbde-sync")
|
|
assert.equal(files[0]?.mode, 0o700)
|
|
}
|
|
service["runStreamingCommandWithTransientSshRetry"] = async (
|
|
ssh: { host: string },
|
|
nextCommand: string,
|
|
streamOutput: boolean
|
|
) => {
|
|
assert.equal(ssh.host, "203.0.113.10")
|
|
command = nextCommand
|
|
assert.equal(streamOutput, true)
|
|
}
|
|
|
|
await service.reconcileNbdeNode({
|
|
server: createServerRecord(),
|
|
managedBootstrap: {
|
|
nodePrivateKey: "a".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
},
|
|
nbde: {
|
|
mode: "degraded",
|
|
luksUuid: "test-luks-uuid",
|
|
recoverySecret: "test-recovery-secret",
|
|
localBootKey: "test-local-boot-key",
|
|
peerTangUrls: []
|
|
}
|
|
} as never)
|
|
|
|
assert.equal(uploaded, true)
|
|
assert.match(command ?? "", /LEGION_NBDE_MODE='degraded'/)
|
|
assert.match(command ?? "", /LEGION_NBDE_LUKS_UUID='test-luks-uuid'/)
|
|
assert.match(command ?? "", /sh \/tmp\/legion\/nbde-sync/)
|
|
})
|
|
|
|
test("waitForNodeReady returns the observed pubkey for managed bootstrap nodes", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
|
|
service["waitForTangReady"] = async () => {}
|
|
service["waitForBootstrapReady"] = async () => {}
|
|
service["readNodePublicKey"] = async () => "a".repeat(64)
|
|
service["readNbdeLuksUuid"] = async () => "uuid-1"
|
|
|
|
const result = await service.waitForNodeReady(session)
|
|
|
|
assert.equal(result.nodePublicKey, "a".repeat(64))
|
|
assert.equal(result.nbde.luksUuid, "uuid-1")
|
|
})
|
|
|
|
test("LEGION_TEST_CERT_MODE=self-signed forces self-signed deployment profiles", async () => {
|
|
const previousMode = process.env["LEGION_TEST_CERT_MODE"]
|
|
process.env["LEGION_TEST_CERT_MODE"] = "self-signed"
|
|
|
|
try {
|
|
const service = new NodeDeploymentService()
|
|
const sshPublicKey =
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBkGzctnVw9laH2L8N8Pqj9QY0aY4vS2d15YvH8lU2rA"
|
|
const session = await service.prepareReconfigureSession({
|
|
server: createServerRecord({
|
|
sshPublicKey
|
|
}),
|
|
bootstrapMode: "init",
|
|
managedBootstrap: {
|
|
nodePrivateKey: "a".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
},
|
|
publicHost: "203.0.113.10",
|
|
certificateName: "203-0-113-10",
|
|
certificateSubjects: ["203.0.113.10"],
|
|
certificateChallengeMode: "http",
|
|
desiredClusterMembership: [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
}
|
|
],
|
|
clusterTargets: [],
|
|
nbde: {
|
|
mode: "degraded",
|
|
tangPort: 7654,
|
|
recoverySecret: "secret",
|
|
localBootKeyPresent: true,
|
|
peerTangNodeIds: [],
|
|
peerTangUrls: []
|
|
},
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
tribeName: "Test Tribe",
|
|
tribeDescription: "Test tribe",
|
|
tribeVisibility: "invite_only",
|
|
kexecImagePath: "/tmp/guix-kexec-installer.tar.gz",
|
|
bootMode: "bios"
|
|
})
|
|
|
|
assert.equal(session.deploymentProfile.edge.certificateProfile, "self-signed")
|
|
assert.equal(session.hostConfig.edge.certificateProfile, "self-signed")
|
|
} finally {
|
|
if (typeof previousMode === "string") {
|
|
process.env["LEGION_TEST_CERT_MODE"] = previousMode
|
|
} else {
|
|
delete process.env["LEGION_TEST_CERT_MODE"]
|
|
}
|
|
}
|
|
})
|
|
|
|
test("join deployment profile uses resolved rollout plugins from upstream", async () => {
|
|
const service = new NodeDeploymentService()
|
|
const internals = service as unknown as Record<string, unknown>
|
|
const requestedTarget = {
|
|
channels: [
|
|
{
|
|
channel_id: "channel-1",
|
|
commit: "commit-a",
|
|
position: 10
|
|
}
|
|
],
|
|
plugins: [
|
|
{
|
|
plugin_name: "tribe-one-kobold",
|
|
channel_id: null,
|
|
enabled: true
|
|
}
|
|
],
|
|
rolloutPolicy: {
|
|
prepare_timeout_seconds: 1800,
|
|
commit_timeout_seconds: 300,
|
|
require_all_nodes_ready: true
|
|
}
|
|
}
|
|
|
|
internals["waitForPublicAdminEndpointReady"] = async () => {}
|
|
internals["createAdminApiClient"] = () => ({
|
|
clusterPreviewRollout: async () => ({
|
|
ok: true,
|
|
schema_version: "1",
|
|
mode: "auto_switch",
|
|
target: requestedTarget,
|
|
preview: {},
|
|
plan: {
|
|
plan_schema_version: "1",
|
|
plan_hash: "plan-test",
|
|
resolved_channels: [
|
|
{
|
|
channel_id: "channel-1",
|
|
name: "tribes",
|
|
url: "https://git.example.test/guix-tribes.git",
|
|
branch: "master",
|
|
commit: "commit-a",
|
|
introduction: {
|
|
commit: "intro-a",
|
|
fingerprint: "FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"
|
|
},
|
|
position: 10
|
|
}
|
|
],
|
|
resolved_plugins: [
|
|
{
|
|
name: "tribe-one-trust",
|
|
enabled: true
|
|
},
|
|
{
|
|
name: "tribe-one-kobold",
|
|
enabled: true
|
|
}
|
|
],
|
|
resolved_extra_packages: [],
|
|
core_migration_target: false,
|
|
core_destructive_rollback_migrations: [],
|
|
closure_estimate_bytes: false
|
|
}
|
|
}),
|
|
clusterUpdateSourcesExport: async () => ({
|
|
ok: true,
|
|
schema_version: "1",
|
|
generated_at: "2026-06-18T00:00:00.000Z",
|
|
update_defaults: {
|
|
trusted_signers: [
|
|
{
|
|
id: "signer-id-1",
|
|
label: "cluster signer",
|
|
fingerprint: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
|
|
enabled: true
|
|
}
|
|
],
|
|
channels: [
|
|
{
|
|
id: "channel-id-1",
|
|
name: "tribes",
|
|
url: "https://git.example.test/guix-tribes.git",
|
|
branch: "master",
|
|
tracked_commit: "commit-a",
|
|
introduction: {
|
|
commit: "intro-a",
|
|
fingerprint: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
|
|
},
|
|
allowed_signer_ids: ["signer-id-1"],
|
|
position: 10,
|
|
enabled: true
|
|
}
|
|
],
|
|
substitute_servers: [
|
|
{
|
|
id: "server-id-1",
|
|
name: "guix.example.test",
|
|
url: "https://guix.example.test",
|
|
position: 10,
|
|
enabled: true
|
|
}
|
|
],
|
|
substitute_keys: [
|
|
{
|
|
id: "key-id-1",
|
|
label: "guix.example.test",
|
|
public_key: "guix.example.test:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
|
enabled: true
|
|
}
|
|
]
|
|
}
|
|
})
|
|
})
|
|
|
|
const sshPublicKey =
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBkGzctnVw9laH2L8N8Pqj9QY0aY4vS2d15YvH8lU2rA"
|
|
const session = await service.prepareReconfigureSession({
|
|
server: createServerRecord({ id: "node-b", publicIp: "203.0.113.11", sshPublicKey }),
|
|
bootstrapMode: "join",
|
|
managedBootstrap: {
|
|
nodePrivateKey: "b".repeat(64),
|
|
publicIpv4: "203.0.113.11",
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
},
|
|
publicHost: "203.0.113.11",
|
|
certificateName: "203-0-113-11",
|
|
certificateSubjects: ["203.0.113.11"],
|
|
certificateChallengeMode: "http",
|
|
upstream: {
|
|
server: createServerRecord({ id: "node-a", sshPublicKey }),
|
|
managedBootstrap: {
|
|
nodePrivateKey: "a".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
}
|
|
},
|
|
desiredClusterMembership: [
|
|
{
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "https://203.0.113.10:4413",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-05-28T00:00:00.000Z"
|
|
}
|
|
],
|
|
clusterTargets: [],
|
|
nbde: {
|
|
mode: "degraded",
|
|
tangPort: 7654,
|
|
recoverySecret: "secret",
|
|
localBootKeyPresent: true,
|
|
peerTangNodeIds: [],
|
|
peerTangUrls: []
|
|
},
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
tribeName: "Test Tribe",
|
|
tribeDescription: "Test tribe",
|
|
tribeVisibility: "invite_only",
|
|
bootMode: "bios"
|
|
})
|
|
|
|
const bootstrap = JSON.parse(session.bootstrapJson)
|
|
|
|
assert.deepEqual(session.deploymentProfile.plugins, ["tribe-one-kobold", "tribe-one-trust"])
|
|
assert.deepEqual(session.hostConfig.tribes.plugins, ["tribe-one-kobold", "tribe-one-trust"])
|
|
assert.equal(session.deploymentProfile.channel?.url, "https://git.example.test/guix-tribes.git")
|
|
assert.equal(session.deploymentProfile.channel?.commit, "commit-a")
|
|
assert.equal(bootstrap.update_defaults.trusted_signers[0].id, "signer-id-1")
|
|
assert.equal(bootstrap.update_defaults.channels[0].id, "channel-id-1")
|
|
assert.deepEqual(bootstrap.update_defaults.channels[0].allowed_signer_ids, ["signer-id-1"])
|
|
assert.equal(bootstrap.update_defaults.substitute_servers[0].id, "server-id-1")
|
|
assert.equal(bootstrap.update_defaults.substitute_keys[0].id, "key-id-1")
|
|
})
|
|
|
|
test("summarizeAcmeCertificateIssue reports ACME rate limits explicitly", () => {
|
|
const summary = summarizeAcmeCertificateIssue(
|
|
"203.0.113.10",
|
|
"lego: ACME rate-limited; skipping immediate retries and keeping existing/self-signed certificate.\nacme:error:ratelimited: too many certificates already issued for this exact set of identifiers"
|
|
)
|
|
|
|
assert.equal(
|
|
summary,
|
|
"ACME rate limit detected for 203.0.113.10: acme:error:ratelimited: too many certificates already issued for this exact set of identifiers. 203.0.113.10 kept serving the bootstrap self-signed certificate because no fresh certificate was issued."
|
|
)
|
|
})
|
|
|
|
test("summarizeAcmeCertificateIssue reports specific ACME failures", () => {
|
|
const summary = summarizeAcmeCertificateIssue(
|
|
"203.0.113.10",
|
|
"Error: one or more domains had a problem:\n[203.0.113.10] authorization failed: HTTP 400 urn:ietf:params:acme:error:connection: timeout during connect"
|
|
)
|
|
|
|
assert.equal(
|
|
summary,
|
|
"ACME certificate provisioning failed for 203.0.113.10: [203.0.113.10] authorization failed: HTTP 400 urn:ietf:params:acme:error:connection: timeout during connect. HAProxy may still be serving the bootstrap self-signed certificate."
|
|
)
|
|
})
|
|
|
|
test("summarizeAcmeCertificateIssue falls back to the latest lego line when parsing is inconclusive", () => {
|
|
const summary = summarizeAcmeCertificateIssue(
|
|
"203.0.113.10",
|
|
"mode=run\nstatus=ok\ncertificate updated"
|
|
)
|
|
|
|
assert.equal(
|
|
summary,
|
|
"ACME certificate diagnostics for 203.0.113.10 were inconclusive. Latest lego output: certificate updated. The node still served the bootstrap self-signed certificate after provisioning, so certificate installation or reload may have failed."
|
|
)
|
|
})
|
|
|
|
test("waitForNodeReady publishes join membership to existing targets before waiting for sync", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
const request = session.request as NodeDeploymentInstallSession["request"] &
|
|
Record<string, unknown>
|
|
|
|
const joinTarget = {
|
|
nodePrivateKey: "a".repeat(64),
|
|
nodePublicKey: "a".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
const upstreamTarget = {
|
|
nodePrivateKey: "c".repeat(64),
|
|
nodePublicKey: "c".repeat(64),
|
|
publicIpv4: "203.0.113.20",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
|
|
request.bootstrapMode = "join"
|
|
request.managedBootstrap = joinTarget
|
|
request.clusterBootstrap = createTestClusterBootstrap()
|
|
request.desiredClusterMembership = [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
},
|
|
{
|
|
nodeId: "node-upstream",
|
|
pubkey: "c".repeat(64),
|
|
transportAddress: "wss://203.0.113.20:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-09T00:00:00.000Z"
|
|
}
|
|
]
|
|
request.clusterTargets = [
|
|
{
|
|
managedBootstrap: joinTarget,
|
|
server: createServerRecord({
|
|
id: "node-a",
|
|
publicIp: "203.0.113.10",
|
|
sshPrivateKey: "join-key"
|
|
})
|
|
},
|
|
{
|
|
managedBootstrap: upstreamTarget,
|
|
server: createServerRecord({
|
|
id: "node-upstream",
|
|
publicIp: "203.0.113.20",
|
|
sshPrivateKey: "upstream-key"
|
|
})
|
|
}
|
|
]
|
|
|
|
let syncRequest: ClusterMembershipSyncRequest | undefined
|
|
const restartCommands: string[] = []
|
|
const order: string[] = []
|
|
service["waitForTangReady"] = async () => {}
|
|
service["syncClusterMembership"] = async (input: ClusterMembershipSyncRequest) => {
|
|
order.push("sync")
|
|
syncRequest = input
|
|
}
|
|
service["waitForClusterMeshPeerReady"] = async (input: { peerPubkey: string }) => {
|
|
order.push("mesh")
|
|
assert.equal(input.peerPubkey, "a".repeat(64))
|
|
}
|
|
service["runStreamingCommand"] = async (_ssh: unknown, command: string) => {
|
|
order.push("restart")
|
|
restartCommands.push(command)
|
|
}
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
service["waitForBootstrapPublicationBarrier"] = async () => {
|
|
order.push("barrier")
|
|
}
|
|
service["readNodePublicKey"] = async () => "a".repeat(64)
|
|
service["readNbdeLuksUuid"] = async () => "uuid-join"
|
|
|
|
const result = await service.waitForNodeReady(session)
|
|
|
|
assert.equal(result.nodePublicKey, "a".repeat(64))
|
|
assert.equal(result.nbde.luksUuid, "uuid-join")
|
|
assert.ok(syncRequest)
|
|
assert.deepEqual(syncRequest.clusterTargets, [upstreamTarget])
|
|
assert.deepEqual(syncRequest.preflightTargets, [upstreamTarget])
|
|
assert.equal(syncRequest.preserveExistingMembers, true)
|
|
assert.equal(syncRequest.allowSyncingForActive, true)
|
|
assert.deepEqual(order, ["sync", "mesh", "restart", "barrier"])
|
|
assert.deepEqual(restartCommands, [
|
|
"herd restart tribes || { herd stop tribes || true; herd start tribes; }"
|
|
])
|
|
})
|
|
|
|
test("reconcileClusterMembership allows syncing views for active members", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
const request = session.request as NodeDeploymentInstallSession["request"] &
|
|
Record<string, unknown>
|
|
const target = {
|
|
nodePrivateKey: "a".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
let syncRequest: ClusterMembershipSyncRequest | undefined
|
|
|
|
request.bootstrapMode = "join"
|
|
request.managedBootstrap = target
|
|
request.clusterBootstrap = createTestClusterBootstrap()
|
|
request.clusterTargets = [
|
|
{
|
|
managedBootstrap: target,
|
|
server: createServerRecord({ id: "node-a", publicIp: "203.0.113.10" })
|
|
}
|
|
]
|
|
service["syncClusterMembership"] = async (input: ClusterMembershipSyncRequest) => {
|
|
syncRequest = input
|
|
}
|
|
|
|
await service.reconcileClusterMembership(session)
|
|
|
|
assert.ok(syncRequest)
|
|
assert.equal(syncRequest.allowSyncingForActive, true)
|
|
})
|
|
|
|
test("syncClusterMembership preserves members discovered during convergence", async () => {
|
|
const service = new NodeDeploymentService(undefined, {
|
|
delay: immediateDelay
|
|
}) as NodeDeploymentService & Record<string, unknown>
|
|
const targetA = createClusterMembershipTarget("a", "203.0.113.10")
|
|
const targetB = createClusterMembershipTarget("b", "203.0.113.11")
|
|
const nodeA = createClusterMembershipRecord("a", "203.0.113.10")
|
|
const nodeB = createClusterMembershipRecord("b", "203.0.113.11")
|
|
const nodeC = createClusterMembershipRecord("c", "203.0.113.12")
|
|
const viewsByHost = new Map<string, ClusterMembershipRecord[]>([
|
|
["203.0.113.10", [nodeA, nodeB, nodeC]],
|
|
["203.0.113.11", [nodeA, nodeB]]
|
|
])
|
|
const upserts: Array<{ host: string; pubkey: string; status?: string }> = []
|
|
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
;(service as unknown as ServiceTestInternals).createAdminApiClient = (_request, target) => ({
|
|
clusterNodeUpsert: async (input) => {
|
|
const host = target.publicIpv4 ?? "unknown"
|
|
const current = viewsByHost.get(host) ?? []
|
|
const nextEntry: ClusterMembershipRecord = {
|
|
pubkey: input.pubkey,
|
|
transportAddress: input.transportAddress ?? `wss://${host}:4413/relay`,
|
|
scope: input.scope ?? "all",
|
|
status: input.status ?? "active",
|
|
activatedAt: input.activatedAt ?? "2026-04-06T12:00:00.000Z",
|
|
deactivatedAt: input.deactivatedAt
|
|
}
|
|
viewsByHost.set(host, [
|
|
...current.filter((entry) => entry.pubkey !== input.pubkey),
|
|
nextEntry
|
|
])
|
|
upserts.push({ host, pubkey: input.pubkey, status: input.status })
|
|
},
|
|
clusterNodesList: async () => ({
|
|
nodes: (viewsByHost.get(target.publicIpv4 ?? "unknown") ?? []).map((entry) => ({
|
|
pubkey: entry.pubkey,
|
|
transport_address: entry.transportAddress,
|
|
scope: entry.scope,
|
|
status: entry.status,
|
|
activated_at: entry.activatedAt,
|
|
deactivated_at: entry.deactivatedAt ?? null
|
|
}))
|
|
})
|
|
})
|
|
|
|
await service.syncClusterMembership({
|
|
desiredClusterMembership: [nodeA, nodeB],
|
|
clusterTargets: [targetA, targetB],
|
|
preflightTargets: [],
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
preserveExistingMembers: true
|
|
})
|
|
|
|
assert.ok(upserts.some((entry) => entry.host === "203.0.113.11" && entry.pubkey === nodeC.pubkey))
|
|
assert.deepEqual(
|
|
[...(viewsByHost.get("203.0.113.11") ?? [])].map((entry) => entry.pubkey).sort(),
|
|
[nodeA.pubkey, nodeB.pubkey, nodeC.pubkey].sort()
|
|
)
|
|
})
|
|
|
|
test("syncClusterMembership keeps desired decommissioned status over observed active members", async () => {
|
|
const service = new NodeDeploymentService(undefined, {
|
|
delay: immediateDelay
|
|
}) as NodeDeploymentService & Record<string, unknown>
|
|
const target = createClusterMembershipTarget("a", "203.0.113.10")
|
|
const nodeA = createClusterMembershipRecord("a", "203.0.113.10")
|
|
const failedNode = createClusterMembershipRecord("b", "203.0.113.11")
|
|
const desiredFailedNode = {
|
|
...failedNode,
|
|
status: "decommissioned" as const,
|
|
deactivatedAt: "2026-04-06T13:00:00.000Z"
|
|
}
|
|
const view = [nodeA, failedNode]
|
|
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
;(service as unknown as ServiceTestInternals).createAdminApiClient = () => ({
|
|
clusterNodeUpsert: async (input) => {
|
|
const nextEntry: ClusterMembershipRecord = {
|
|
pubkey: input.pubkey,
|
|
transportAddress: input.transportAddress ?? `wss://203.0.113.10:4413/relay`,
|
|
scope: input.scope ?? "all",
|
|
status: input.status ?? "active",
|
|
activatedAt: input.activatedAt ?? "2026-04-06T12:00:00.000Z",
|
|
deactivatedAt: input.deactivatedAt
|
|
}
|
|
const index = view.findIndex((entry) => entry.pubkey === input.pubkey)
|
|
if (index >= 0) {
|
|
view[index] = nextEntry
|
|
} else {
|
|
view.push(nextEntry)
|
|
}
|
|
},
|
|
clusterNodesList: async () => ({
|
|
nodes: view.map((entry) => ({
|
|
pubkey: entry.pubkey,
|
|
transport_address: entry.transportAddress,
|
|
scope: entry.scope,
|
|
status: entry.status,
|
|
activated_at: entry.activatedAt,
|
|
deactivated_at: entry.deactivatedAt ?? null
|
|
}))
|
|
})
|
|
})
|
|
|
|
await service.syncClusterMembership({
|
|
desiredClusterMembership: [nodeA, desiredFailedNode],
|
|
clusterTargets: [target],
|
|
preflightTargets: [],
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
preserveExistingMembers: true
|
|
})
|
|
|
|
assert.equal(view.find((entry) => entry.pubkey === failedNode.pubkey)?.status, "decommissioned")
|
|
})
|
|
|
|
test("waitForClusterMeshPeerReady observes mesh status for every existing target", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const targets = [
|
|
{
|
|
nodePrivateKey: "1".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
},
|
|
{
|
|
nodePrivateKey: "2".repeat(64),
|
|
publicIpv4: "203.0.113.11",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
]
|
|
const statusReads: string[] = []
|
|
const serviceInternals = service as unknown as ServiceTestInternals
|
|
|
|
serviceInternals.createAdminApiClient = (_request, target) => ({
|
|
clusterNodeUpsert: async () => {},
|
|
clusterStatus: async () => {
|
|
statusReads.push(target.publicIpv4 ?? "unknown")
|
|
return {
|
|
servers: [{ auth_pubkey: "A".repeat(64) }]
|
|
}
|
|
}
|
|
})
|
|
|
|
await service["waitForClusterMeshPeerReady"]({
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
targets,
|
|
peerPubkey: "a".repeat(64)
|
|
})
|
|
|
|
assert.deepEqual(statusReads, ["203.0.113.10", "203.0.113.11"])
|
|
})
|
|
|
|
test("waitForNodeReady rejects managed bootstrap pubkey mismatches", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
|
|
service["waitForTangReady"] = async () => {}
|
|
service["waitForBootstrapReady"] = async () => {}
|
|
service["readNodePublicKey"] = async () => "b".repeat(64)
|
|
service["readNbdeLuksUuid"] = async () => "uuid-1"
|
|
|
|
await assert.rejects(
|
|
service.waitForNodeReady(session),
|
|
/Node pubkey mismatch after managed bootstrap: expected a{64}, observed b{64}\./
|
|
)
|
|
})
|
|
|
|
test("syncClusterMembership writes the desired snapshot to every target and verifies convergence", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const desiredClusterMembership = [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all" as const,
|
|
status: "active" as const,
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
},
|
|
{
|
|
nodeId: "node-b",
|
|
pubkey: "b".repeat(64),
|
|
transportAddress: "wss://203.0.113.11:4413/relay",
|
|
scope: "all" as const,
|
|
status: "decommissioned" as const,
|
|
activatedAt: "2026-04-09T00:00:00.000Z",
|
|
deactivatedAt: "2026-04-11T00:00:00.000Z"
|
|
}
|
|
]
|
|
const clusterTargets = [
|
|
{
|
|
nodePrivateKey: "1".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
},
|
|
{
|
|
nodePrivateKey: "2".repeat(64),
|
|
publicIpv4: "203.0.113.11",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
]
|
|
const upserts: Array<{ host: string; pubkey: string }> = []
|
|
let viewReads = 0
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
const serviceInternals = service as unknown as ServiceTestInternals
|
|
|
|
serviceInternals.createAdminApiClient = (
|
|
request: { clusterBootstrap?: unknown },
|
|
target: { publicIpv4?: string }
|
|
) => {
|
|
assert.ok(request.clusterBootstrap)
|
|
|
|
return {
|
|
clusterNodeUpsert: async (input: { pubkey: string }) => {
|
|
upserts.push({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
pubkey: input.pubkey
|
|
})
|
|
}
|
|
}
|
|
}
|
|
serviceInternals.readClusterMembershipViews = async (
|
|
request: { clusterBootstrap?: unknown },
|
|
targets: Array<{ publicIpv4?: string }>
|
|
) => {
|
|
assert.ok(request.clusterBootstrap)
|
|
viewReads += 1
|
|
|
|
if (viewReads === 1) {
|
|
return targets.map((target) => ({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
nodes: [desiredClusterMembership[0]]
|
|
}))
|
|
}
|
|
|
|
return targets.map((target) => ({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
nodes: desiredClusterMembership
|
|
}))
|
|
}
|
|
|
|
await service.syncClusterMembership({
|
|
desiredClusterMembership,
|
|
clusterTargets,
|
|
preflightTargets: [clusterTargets[0]],
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin()
|
|
})
|
|
|
|
assert.equal(viewReads, 2)
|
|
assert.deepEqual(upserts, [
|
|
{ host: "203.0.113.10", pubkey: "a".repeat(64) },
|
|
{ host: "203.0.113.10", pubkey: "b".repeat(64) },
|
|
{ host: "203.0.113.11", pubkey: "a".repeat(64) },
|
|
{ host: "203.0.113.11", pubkey: "b".repeat(64) }
|
|
])
|
|
})
|
|
|
|
test("syncClusterMembership can preserve peers already present in a preflight view", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const desiredClusterMembership: ClusterMembershipRecord[] = [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
},
|
|
{
|
|
nodeId: "node-c",
|
|
pubkey: "c".repeat(64),
|
|
transportAddress: "wss://203.0.113.12:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-12T00:00:00.000Z"
|
|
}
|
|
]
|
|
const existingPeer: ClusterMembershipRecord = {
|
|
nodeId: "node-b",
|
|
pubkey: "b".repeat(64),
|
|
transportAddress: "wss://203.0.113.11:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-11T00:00:00.000Z"
|
|
}
|
|
const mergedMembership = [
|
|
desiredClusterMembership[0]!,
|
|
existingPeer,
|
|
desiredClusterMembership[1]!
|
|
]
|
|
const clusterTargets = [
|
|
{
|
|
nodePrivateKey: "1".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
]
|
|
const upserts: string[] = []
|
|
let viewReads = 0
|
|
const serviceInternals = service as unknown as ServiceTestInternals
|
|
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
serviceInternals.createAdminApiClient = () => ({
|
|
clusterNodeUpsert: async (input: { pubkey: string }) => {
|
|
upserts.push(input.pubkey)
|
|
}
|
|
})
|
|
serviceInternals.readClusterMembershipViews = async (
|
|
_request: { clusterBootstrap?: unknown },
|
|
targets: Array<{ publicIpv4?: string }>
|
|
) => {
|
|
viewReads += 1
|
|
|
|
if (viewReads === 1) {
|
|
return targets.map((target) => ({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
nodes: [desiredClusterMembership[0]!, existingPeer]
|
|
}))
|
|
}
|
|
|
|
return targets.map((target) => ({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
nodes: mergedMembership
|
|
}))
|
|
}
|
|
|
|
await service.syncClusterMembership({
|
|
desiredClusterMembership,
|
|
clusterTargets,
|
|
preflightTargets: clusterTargets,
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
preserveExistingMembers: true
|
|
})
|
|
|
|
assert.equal(viewReads, 2)
|
|
assert.deepEqual(upserts.sort(), ["a".repeat(64), "b".repeat(64), "c".repeat(64)])
|
|
})
|
|
|
|
test("syncClusterMembership accepts syncing observations for active join requests when allowed", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const desiredClusterMembership: ClusterMembershipRecord[] = [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
}
|
|
]
|
|
const observedClusterMembership: ClusterMembershipRecord[] = [
|
|
{
|
|
...desiredClusterMembership[0]!,
|
|
status: "syncing"
|
|
}
|
|
]
|
|
const clusterTargets = [
|
|
{
|
|
nodePrivateKey: "1".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
]
|
|
let viewReads = 0
|
|
const serviceInternals = service as unknown as ServiceTestInternals
|
|
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
serviceInternals.createAdminApiClient = () => ({
|
|
clusterNodeUpsert: async () => {}
|
|
})
|
|
serviceInternals.readClusterMembershipViews = async (
|
|
_request: { clusterBootstrap?: unknown },
|
|
targets: Array<{ publicIpv4?: string }>
|
|
) => {
|
|
viewReads += 1
|
|
|
|
return targets.map((target) => ({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
nodes: observedClusterMembership
|
|
}))
|
|
}
|
|
|
|
await service.syncClusterMembership({
|
|
desiredClusterMembership,
|
|
clusterTargets,
|
|
preflightTargets: [],
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
allowSyncingForActive: true
|
|
})
|
|
|
|
assert.equal(viewReads, 2)
|
|
})
|
|
|
|
test("syncClusterMembership accepts divergent syncing views for active join requests when allowed", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const desiredClusterMembership: ClusterMembershipRecord[] = [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
},
|
|
{
|
|
nodeId: "node-b",
|
|
pubkey: "b".repeat(64),
|
|
transportAddress: "wss://203.0.113.11:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-11T00:00:00.000Z"
|
|
}
|
|
]
|
|
const clusterTargets = [
|
|
{
|
|
nodePrivateKey: "1".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
},
|
|
{
|
|
nodePrivateKey: "2".repeat(64),
|
|
publicIpv4: "203.0.113.11",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
]
|
|
let viewReads = 0
|
|
const serviceInternals = service as unknown as ServiceTestInternals
|
|
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
serviceInternals.createAdminApiClient = () => ({
|
|
clusterNodeUpsert: async () => {}
|
|
})
|
|
serviceInternals.readClusterMembershipViews = async () => {
|
|
viewReads += 1
|
|
|
|
return [
|
|
{
|
|
host: "203.0.113.10",
|
|
nodes: desiredClusterMembership
|
|
},
|
|
{
|
|
host: "203.0.113.11",
|
|
nodes: [
|
|
{ ...desiredClusterMembership[0]!, status: "syncing" },
|
|
desiredClusterMembership[1]!
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
await service.syncClusterMembership({
|
|
desiredClusterMembership,
|
|
clusterTargets,
|
|
preflightTargets: [],
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
allowSyncingForActive: true
|
|
})
|
|
|
|
assert.equal(viewReads, 2)
|
|
})
|
|
|
|
test("syncClusterMembership reconciles divergent preflight cluster views", async () => {
|
|
const reports: string[] = []
|
|
const reporter: NodeDeploymentReporter = {
|
|
info: (line) => reports.push(line),
|
|
stdout: () => {},
|
|
stderr: () => {}
|
|
}
|
|
const service = new NodeDeploymentService(reporter) as NodeDeploymentService &
|
|
Record<string, unknown>
|
|
const desiredClusterMembership: ClusterMembershipRecord[] = [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: "a".repeat(64),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
},
|
|
{
|
|
nodeId: "node-b",
|
|
pubkey: "b".repeat(64),
|
|
transportAddress: "wss://203.0.113.11:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-11T00:00:00.000Z"
|
|
},
|
|
{
|
|
nodeId: "node-c",
|
|
pubkey: "c".repeat(64),
|
|
transportAddress: "wss://203.0.113.12:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-12T00:00:00.000Z"
|
|
}
|
|
]
|
|
const clusterTargets = [
|
|
{
|
|
nodePrivateKey: "1".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
},
|
|
{
|
|
nodePrivateKey: "2".repeat(64),
|
|
publicIpv4: "203.0.113.11",
|
|
publicPort: 443,
|
|
publicScheme: "https" as const
|
|
}
|
|
]
|
|
const upserts: Array<{ host: string; pubkey: string }> = []
|
|
let viewReads = 0
|
|
const serviceInternals = service as unknown as ServiceTestInternals
|
|
|
|
service["waitForPublicAdminEndpointReady"] = async () => {}
|
|
serviceInternals.createAdminApiClient = (_request, target) => ({
|
|
clusterNodeUpsert: async (input: { pubkey: string }) => {
|
|
upserts.push({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
pubkey: input.pubkey
|
|
})
|
|
}
|
|
})
|
|
serviceInternals.readClusterMembershipViews = async (
|
|
_request: { clusterBootstrap?: unknown },
|
|
targets: Array<{ publicIpv4?: string }>
|
|
) => {
|
|
viewReads += 1
|
|
|
|
if (viewReads === 1) {
|
|
return targets.map((target, index) => ({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
nodes:
|
|
index === 0
|
|
? desiredClusterMembership
|
|
: [desiredClusterMembership[0]!, desiredClusterMembership[1]!]
|
|
}))
|
|
}
|
|
|
|
return targets.map((target) => ({
|
|
host: target.publicIpv4 ?? "unknown",
|
|
nodes: desiredClusterMembership
|
|
}))
|
|
}
|
|
|
|
await service.syncClusterMembership({
|
|
desiredClusterMembership,
|
|
clusterTargets,
|
|
preflightTargets: clusterTargets,
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin()
|
|
})
|
|
|
|
assert.equal(viewReads, 2)
|
|
assert.equal(upserts.length, 6)
|
|
assert.match(reports.join("\n"), /cluster membership preflight: observed divergent views/)
|
|
})
|
|
|
|
test("parseBootstrapProbeState recognizes ready output", () => {
|
|
assert.deepEqual(
|
|
parseBootstrapProbeState(
|
|
'{"health":"ok","bootstrap":{"managed":true,"ready":true,"waiting":false,"terminal":false,"status":"ready","detail":null},"node":{"pubkey":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}\n'
|
|
),
|
|
{
|
|
kind: "ready"
|
|
}
|
|
)
|
|
})
|
|
|
|
test("parseBootstrapProbeState recognizes terminal failed output", () => {
|
|
assert.deepEqual(
|
|
parseBootstrapProbeState(
|
|
'{"health":"ok","bootstrap":{"managed":true,"ready":false,"waiting":true,"terminal":true,"status":"failed","detail":"singleton tribe already exists"},"node":{"pubkey":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}\n'
|
|
),
|
|
{
|
|
kind: "failed",
|
|
reason: "singleton tribe already exists"
|
|
}
|
|
)
|
|
})
|
|
|
|
test("parseBootstrapProbeState treats other output as waiting", () => {
|
|
assert.deepEqual(
|
|
parseBootstrapProbeState(
|
|
'{"health":"ok","bootstrap":{"managed":true,"ready":false,"waiting":true,"terminal":false,"status":"bootstrapping","detail":null},"node":{"pubkey":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}\n'
|
|
),
|
|
{
|
|
kind: "waiting"
|
|
}
|
|
)
|
|
})
|
|
|
|
test("parseInternalStatusNodePublicKey extracts the lowercase pubkey from internal status JSON", () => {
|
|
assert.equal(
|
|
parseInternalStatusNodePublicKey(
|
|
'{"health":"ok","bootstrap":{"managed":true,"ready":true,"waiting":false,"terminal":false,"status":"ready","detail":null},"node":{"pubkey":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}}\n'
|
|
),
|
|
"a".repeat(64)
|
|
)
|
|
})
|
|
|
|
test("parseInternalStatusNodePublicKey returns null for invalid payloads", () => {
|
|
assert.equal(parseInternalStatusNodePublicKey("not json\n"), null)
|
|
})
|
|
|
|
test("parseLegionDeploymentStatusMarker recognizes fixed deployment status lines", () => {
|
|
assert.deepEqual(
|
|
parseLegionDeploymentStatusMarker(
|
|
"@LEGION_STATUS v=1 phase=guix-install step=system-init state=started\n"
|
|
),
|
|
{
|
|
version: 1,
|
|
phase: "guix-install",
|
|
step: "system-init",
|
|
state: "started"
|
|
}
|
|
)
|
|
assert.equal(parseLegionDeploymentStatusMarker("@LEGION_STATUS v=2 phase=x step=y state=z"), null)
|
|
assert.equal(parseLegionDeploymentStatusMarker("ordinary output"), null)
|
|
})
|
|
|
|
test("prepareInstallSession derives the session pubkey from the stored private key", async () => {
|
|
const tempDir = await mkdtemp(join(tmpdir(), "node-deployment-service-"))
|
|
const kexecImagePath = join(tempDir, "guix-kexec-installer.tar.gz")
|
|
const reporterLines: string[] = []
|
|
const reporter: NodeDeploymentReporter = {
|
|
info(line) {
|
|
reporterLines.push(line)
|
|
},
|
|
stdout(chunk) {
|
|
void chunk
|
|
},
|
|
stderr(chunk) {
|
|
void chunk
|
|
}
|
|
}
|
|
|
|
try {
|
|
await writeFile(kexecImagePath, "")
|
|
|
|
const service = new NodeDeploymentService(reporter)
|
|
const nodePrivateKey = "1".repeat(64)
|
|
const derivedNodePublicKey = deriveNodePublicKey(nodePrivateKey)
|
|
const session = await service.prepareInstallSession({
|
|
server: {
|
|
id: "node-a",
|
|
provider: "manual",
|
|
role: "primary",
|
|
publicIp: "203.0.113.10",
|
|
sshUsername: "root",
|
|
sshPrivateKey: "private-key",
|
|
sshPublicKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIexample",
|
|
status: "running",
|
|
region: "local",
|
|
planName: "manual",
|
|
commercialMetadata: {
|
|
effectiveMonthlyGrossEur: 0,
|
|
source: "override",
|
|
notes: []
|
|
},
|
|
deployment: {
|
|
status: "running"
|
|
},
|
|
createdAt: "2026-04-10T00:00:00.000Z",
|
|
updatedAt: "2026-04-10T00:00:00.000Z"
|
|
},
|
|
managedBootstrap: {
|
|
nodePrivateKey,
|
|
nodePublicKey: "f".repeat(64),
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
},
|
|
publicHost: "203.0.113.10",
|
|
certificateName: "203-0-113-10",
|
|
certificateSubjects: ["203.0.113.10"],
|
|
certificateChallengeMode: "http",
|
|
desiredClusterMembership: [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: derivedNodePublicKey,
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
}
|
|
],
|
|
clusterTargets: [],
|
|
nbde: {
|
|
mode: "degraded",
|
|
tangPort: 7654,
|
|
recoverySecret: "secret",
|
|
localBootKeyPresent: true,
|
|
peerTangNodeIds: [],
|
|
peerTangUrls: []
|
|
},
|
|
clusterBootstrap: {
|
|
...createTestClusterBootstrap(),
|
|
relayPrivateKey: "9".repeat(64)
|
|
},
|
|
tribeName: "Tribe One",
|
|
tribeDescription: "Test tribe",
|
|
tribeVisibility: "invite_only",
|
|
kexecImagePath,
|
|
bootMode: "bios"
|
|
} as unknown as Parameters<NodeDeploymentService["prepareInstallSession"]>[0])
|
|
|
|
assert.equal(session.nodePublicKey, derivedNodePublicKey)
|
|
const bootstrap = JSON.parse(session.bootstrapJson ?? "{}")
|
|
assert.equal(bootstrap.cluster_secret.relay_private_key, "9".repeat(64))
|
|
assert.equal(bootstrap.tribe_admin.username, "legion-admin")
|
|
assert.equal(bootstrap.human_admin, undefined)
|
|
assert.equal(bootstrap.update_defaults.channels[0].name, "tribes")
|
|
assert.equal(
|
|
bootstrap.update_defaults.channels[0].url,
|
|
"https://git.teralink.net/tribes/guix-tribes.git"
|
|
)
|
|
assert.deepEqual(
|
|
bootstrap.update_defaults.substitute_servers.map((server: { url: string }) => server.url),
|
|
["https://guix.tribe-one.org", "https://bordeaux.guix.gnu.org", "https://ci.guix.gnu.org"]
|
|
)
|
|
assert.equal(bootstrap.update_defaults.substitute_keys[0].label, "guix.tribe-one.org")
|
|
assert.match(
|
|
reporterLines.join("\n"),
|
|
/managed bootstrap pubkey mismatch in local state for node-a; using the key derived from the stored private key/
|
|
)
|
|
} finally {
|
|
await rm(tempDir, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
test("prepareReconfigureSession does not require a kexec image path", async () => {
|
|
const service = new NodeDeploymentService()
|
|
const nodePrivateKey = "1".repeat(64)
|
|
|
|
const session = await service.prepareReconfigureSession({
|
|
server: {
|
|
id: "node-a",
|
|
provider: "hetzner",
|
|
role: "primary",
|
|
publicIp: "203.0.113.10",
|
|
sshUsername: "root",
|
|
sshPrivateKey: "private-key",
|
|
sshPublicKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITest legion@test",
|
|
status: "running",
|
|
region: "fsn1",
|
|
planName: "CAX11",
|
|
commercialMetadata: {
|
|
effectiveMonthlyGrossEur: 4,
|
|
source: "provider",
|
|
notes: []
|
|
},
|
|
deployment: {
|
|
status: "ready",
|
|
snippets: []
|
|
},
|
|
createdAt: "2026-04-06T12:00:00.000Z",
|
|
updatedAt: "2026-04-06T12:00:00.000Z"
|
|
},
|
|
bootstrapMode: "init",
|
|
desiredClusterMembership: [
|
|
{
|
|
nodeId: "node-a",
|
|
pubkey: deriveNodePublicKey(nodePrivateKey),
|
|
transportAddress: "wss://203.0.113.10:4413/relay",
|
|
scope: "all",
|
|
status: "active",
|
|
activatedAt: "2026-04-10T00:00:00.000Z"
|
|
}
|
|
],
|
|
clusterTargets: [],
|
|
managedBootstrap: {
|
|
nodePrivateKey,
|
|
nodeName: "node-a",
|
|
publicIpv4: "203.0.113.10",
|
|
publicPort: 443,
|
|
publicScheme: "https"
|
|
},
|
|
publicHost: "203.0.113.10",
|
|
certificateName: "203-0-113-10",
|
|
certificateSubjects: ["203.0.113.10"],
|
|
certificateChallengeMode: "http",
|
|
clusterBootstrap: createTestClusterBootstrap(),
|
|
legionAdmin: createTestLegionAdmin(),
|
|
tribeName: "Test Tribe",
|
|
tribeDescription: "Test tribe",
|
|
tribeVisibility: "invite_only",
|
|
nbde: {
|
|
mode: "degraded",
|
|
tangPort: 7654,
|
|
recoverySecret: "secret",
|
|
localBootKeyPresent: true,
|
|
peerTangNodeIds: [],
|
|
peerTangUrls: []
|
|
},
|
|
kexecImagePath: "/tmp/does-not-need-to-exist.tar.gz",
|
|
bootMode: "bios"
|
|
})
|
|
|
|
assert.equal(session.request.server.id, "node-a")
|
|
assert.equal(session.nodePublicKey, deriveNodePublicKey(nodePrivateKey))
|
|
})
|
|
|
|
test("kexecIntoInstaller is idempotent when the Guix installer is already reachable", async () => {
|
|
const reporterLines: string[] = []
|
|
const service = new NodeDeploymentService({
|
|
info: (line) => reporterLines.push(line),
|
|
stdout: () => {},
|
|
stderr: () => {}
|
|
}) as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
let uploaded = false
|
|
let kexecCommandRun = false
|
|
let waitedForInstaller = false
|
|
|
|
service["isGuixInstallerReachable"] = async () => true
|
|
service["uploadKexecFiles"] = async () => {
|
|
uploaded = true
|
|
}
|
|
service["runStreamingCommand"] = async () => {
|
|
kexecCommandRun = true
|
|
}
|
|
service["waitForGuixInstaller"] = async () => {
|
|
waitedForInstaller = true
|
|
}
|
|
|
|
const result = await service.kexecIntoInstaller(session)
|
|
|
|
assert.equal(result, session)
|
|
assert.equal(uploaded, false)
|
|
assert.equal(kexecCommandRun, false)
|
|
assert.equal(waitedForInstaller, false)
|
|
assert.deepEqual(reporterLines, ["Guix installer is already reachable on 203.0.113.10"])
|
|
})
|
|
|
|
test("kexecIntoInstaller waits for installer identity instead of SSH outage", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
session.request.server.sshPublicKey = "ssh-ed25519 AAAATEST"
|
|
let uploaded = false
|
|
let kexecCommand = ""
|
|
let waitLabel = ""
|
|
|
|
service["isGuixInstallerReachable"] = async () => false
|
|
service["uploadKexecFiles"] = async () => {
|
|
uploaded = true
|
|
}
|
|
service["runStreamingCommand"] = async (_ssh: unknown, command: string) => {
|
|
kexecCommand = command
|
|
}
|
|
service["waitForGuixInstaller"] = async (_ssh: unknown, label: string) => {
|
|
waitLabel = label
|
|
}
|
|
|
|
await service.kexecIntoInstaller(session)
|
|
|
|
assert.equal(uploaded, true)
|
|
assert.match(kexecCommand, /guix-kexec/)
|
|
assert.equal(waitLabel, "waiting for the Guix installer SSH session")
|
|
})
|
|
|
|
test("kexecIntoInstaller stages files with sudo for non-root provider images", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
session.request.server.sshUsername = "ubuntu"
|
|
session.request.server.sshPublicKey = "ssh-ed25519 AAAATEST"
|
|
session.ssh.username = "ubuntu"
|
|
|
|
const uploadedPaths: string[] = []
|
|
const commands: string[] = []
|
|
let installerWaitUser = ""
|
|
|
|
service["isGuixInstallerReachable"] = async () => false
|
|
service["uploadKexecFiles"] = async (_ssh: unknown, files: Array<{ remotePath: string }>) => {
|
|
uploadedPaths.push(...files.map((file) => file.remotePath))
|
|
}
|
|
service["runStreamingCommand"] = async (_ssh: unknown, command: string) => {
|
|
commands.push(command)
|
|
}
|
|
service["waitForGuixInstaller"] = async (ssh: { username: string }) => {
|
|
installerWaitUser = ssh.username
|
|
}
|
|
|
|
const result = await service.kexecIntoInstaller(session)
|
|
|
|
assert.deepEqual(uploadedPaths, [
|
|
"/tmp/legion-kexec/guix-kexec-installer.tar.gz",
|
|
"/tmp/legion-kexec/guix-kexec.sh"
|
|
])
|
|
assert.match(commands[0], /sudo -n sh -c/)
|
|
assert.match(commands[0], /\/root\/kexec\/guix-kexec\.sh/)
|
|
assert.match(commands[1], /^sudo -n sh -c /)
|
|
assert.match(commands[1], /\/root\/kexec\/guix-kexec\.sh/)
|
|
assert.equal(installerWaitUser, "root")
|
|
assert.equal(result.ssh.username, "root")
|
|
assert.equal(result.request.server.sshUsername, "root")
|
|
})
|
|
|
|
test("kexecIntoInstaller retries non-root kexec commands with sudo password", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
session.request.server.sshUsername = "debian"
|
|
session.request.server.sshPublicKey = "ssh-ed25519 AAAATEST"
|
|
session.request.userPassword = "user-secret"
|
|
session.ssh.username = "debian"
|
|
|
|
const commands: Array<{ command: string; stdin?: string | Buffer }> = []
|
|
let installerWaitUser = ""
|
|
|
|
service["isGuixInstallerReachable"] = async () => false
|
|
service["uploadKexecFiles"] = async () => undefined
|
|
service["runStreamingCommand"] = async (
|
|
_ssh: unknown,
|
|
command: string,
|
|
_streamOutput: boolean,
|
|
_allowNonZero?: boolean,
|
|
stdin?: string | Buffer
|
|
) => {
|
|
commands.push({ command, stdin })
|
|
if (command.startsWith("sudo -n ")) {
|
|
throw new Error("sudo: a password is required")
|
|
}
|
|
}
|
|
service["waitForGuixInstaller"] = async (ssh: { username: string }) => {
|
|
installerWaitUser = ssh.username
|
|
}
|
|
|
|
const result = await service.kexecIntoInstaller(session)
|
|
|
|
assert.equal(commands.length, 4)
|
|
assert.match(commands[0]?.command ?? "", /^sudo -n sh -c /)
|
|
assert.match(commands[1]?.command ?? "", /^sudo -S -p '' sh -c /)
|
|
assert.equal(commands[1]?.stdin?.toString(), "user-secret\n")
|
|
assert.doesNotMatch(commands[1]?.command ?? "", /user-secret/)
|
|
assert.match(commands[2]?.command ?? "", /^sudo -n sh -c /)
|
|
assert.match(commands[3]?.command ?? "", /^sudo -S -p '' sh -c /)
|
|
assert.equal(commands[3]?.stdin?.toString(), "user-secret\n")
|
|
assert.doesNotMatch(commands[3]?.command ?? "", /user-secret/)
|
|
assert.equal(installerWaitUser, "root")
|
|
assert.equal(result.ssh.username, "root")
|
|
})
|
|
|
|
test("sudo password retry errors redact the password", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
|
|
service["runStreamingCommand"] = async (_ssh: unknown, command: string) => {
|
|
if (command.startsWith("sudo -n ")) {
|
|
throw new Error("sudo: a password is required")
|
|
}
|
|
|
|
throw new Error("user-secret\ncurl: not found")
|
|
}
|
|
|
|
await assert.rejects(
|
|
() =>
|
|
service["runRootStreamingCommand"](
|
|
{
|
|
host: "203.0.113.10",
|
|
username: "debian"
|
|
},
|
|
"run installer",
|
|
"user-secret"
|
|
) as Promise<void>,
|
|
(error) => {
|
|
assert.ok(error instanceof Error)
|
|
assert.match(error.message, /\[redacted\]/)
|
|
assert.doesNotMatch(error.message, /user-secret/)
|
|
assert.match(error.message, /curl: not found/)
|
|
return true
|
|
}
|
|
)
|
|
})
|
|
|
|
test("rebootIntoInstalledSystem is idempotent when installed Guix is already reachable", async () => {
|
|
const reporterLines: string[] = []
|
|
const service = new NodeDeploymentService({
|
|
info: (line) => reporterLines.push(line),
|
|
stdout: () => {},
|
|
stderr: () => {}
|
|
}) as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
let rebootCommandRun = false
|
|
let waitedForInstalledSystem = false
|
|
|
|
service["isInstalledSystemReachable"] = async () => true
|
|
service["runStreamingCommand"] = async () => {
|
|
rebootCommandRun = true
|
|
}
|
|
service["waitForInstalledSystem"] = async () => {
|
|
waitedForInstalledSystem = true
|
|
}
|
|
|
|
const result = await service.rebootIntoInstalledSystem(session)
|
|
|
|
assert.equal(result, session)
|
|
assert.equal(rebootCommandRun, false)
|
|
assert.equal(waitedForInstalledSystem, false)
|
|
assert.deepEqual(reporterLines, ["installed system is already reachable on 203.0.113.10"])
|
|
})
|
|
|
|
test("rebootIntoInstalledSystem waits for installed system identity instead of SSH outage", async () => {
|
|
const service = new NodeDeploymentService() as NodeDeploymentService & Record<string, unknown>
|
|
const session = createManagedBootstrapSession("a".repeat(64))
|
|
let rebootCommand = ""
|
|
let waitLabel = ""
|
|
|
|
service["isInstalledSystemReachable"] = async () => false
|
|
service["runStreamingCommand"] = async (_ssh: unknown, command: string) => {
|
|
rebootCommand = command
|
|
}
|
|
service["waitForInstalledSystem"] = async (_ssh: unknown, label: string) => {
|
|
waitLabel = label
|
|
}
|
|
|
|
await service.rebootIntoInstalledSystem(session)
|
|
|
|
assert.match(rebootCommand, /reboot/)
|
|
assert.equal(waitLabel, "waiting for the installed system SSH session")
|
|
})
|