fix(v2): make EC2 bootstrap readiness deterministic
This commit is contained in:
@@ -63,22 +63,32 @@ function tryConnect(host: string, privateKey: string, timeoutMs: number): Promis
|
||||
username: "root",
|
||||
privateKey,
|
||||
readyTimeout: timeoutMs,
|
||||
keepaliveInterval: 10000,
|
||||
keepaliveCountMax: 3,
|
||||
algorithms: {
|
||||
serverHostKey: ["ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521"],
|
||||
serverHostKey: ["ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", "ssh-ed25519"],
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const EXEC_TIMEOUT_MS = 30_000;
|
||||
|
||||
function execOnConn(conn: Client, command: string): Promise<{ stdout: string; stderr: string; code: number }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`SSH exec timed out: ${command.slice(0, 60)}`)), EXEC_TIMEOUT_MS);
|
||||
|
||||
conn.exec(command, (err, stream) => {
|
||||
if (err) return reject(err);
|
||||
if (err) {
|
||||
clearTimeout(timer);
|
||||
return reject(err);
|
||||
}
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
stream.on("data", (d: Buffer) => { stdout += d.toString(); });
|
||||
stream.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
|
||||
stream.on("close", (code: number) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ stdout, stderr, code: code ?? 0 });
|
||||
});
|
||||
});
|
||||
@@ -88,8 +98,12 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st
|
||||
export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < maxWaitMs) {
|
||||
const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
|
||||
if (res.stdout.trim() === "done") return;
|
||||
try {
|
||||
const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
|
||||
if (res.stdout.trim() === "done") return;
|
||||
} catch {
|
||||
// transient SSH exec error (e.g. timeout, channel reset) — retry
|
||||
}
|
||||
await sleep(10_000);
|
||||
}
|
||||
throw new Error("EC2 bootstrap did not complete within timeout");
|
||||
|
||||
Reference in New Issue
Block a user