fix(v2): make EC2 bootstrap readiness deterministic

This commit is contained in:
2026-07-25 13:32:20 +00:00
parent 2fb966542b
commit 24c0fa1e7c
3 changed files with 44 additions and 24 deletions
+18 -4
View File
@@ -63,22 +63,32 @@ function tryConnect(host: string, privateKey: string, timeoutMs: number): Promis
username: "root",
privateKey,
readyTimeout: timeoutMs,
keepaliveInterval: 10000,
keepaliveCountMax: 3,
algorithms: {
serverHostKey: ["ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521"],
serverHostKey: ["ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", "ssh-ed25519"],
},
});
});
}
const EXEC_TIMEOUT_MS = 30_000;
function execOnConn(conn: Client, command: string): Promise<{ stdout: string; stderr: string; code: number }> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(`SSH exec timed out: ${command.slice(0, 60)}`)), EXEC_TIMEOUT_MS);
conn.exec(command, (err, stream) => {
if (err) return reject(err);
if (err) {
clearTimeout(timer);
return reject(err);
}
let stdout = "";
let stderr = "";
stream.on("data", (d: Buffer) => { stdout += d.toString(); });
stream.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
stream.on("close", (code: number) => {
clearTimeout(timer);
resolve({ stdout, stderr, code: code ?? 0 });
});
});
@@ -88,8 +98,12 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st
export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise<void> {
const start = Date.now();
while (Date.now() - start < maxWaitMs) {
const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
if (res.stdout.trim() === "done") return;
try {
const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
if (res.stdout.trim() === "done") return;
} catch {
// transient SSH exec error (e.g. timeout, channel reset) — retry
}
await sleep(10_000);
}
throw new Error("EC2 bootstrap did not complete within timeout");