From 24c0fa1e7c918c1cc9850ade6aeb841d6f5e3a87 Mon Sep 17 00:00:00 2001 From: luna Date: Sat, 25 Jul 2026 13:32:20 +0000 Subject: [PATCH] fix(v2): make EC2 bootstrap readiness deterministic --- backend/src/services/deploy.ts | 2 +- backend/src/services/ec2.ts | 44 +++++++++++++++++++--------------- backend/src/services/ssh.ts | 22 +++++++++++++---- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/backend/src/services/deploy.ts b/backend/src/services/deploy.ts index 7bc77d5c..6017c5fc 100644 --- a/backend/src/services/deploy.ts +++ b/backend/src/services/deploy.ts @@ -233,7 +233,7 @@ async function firstDeploy( try { await appendLog(previewId, `[PP] SSH connected. Waiting for bootstrap to complete...\n`); - await waitForBootstrap(sshSession, 600_000); + await waitForBootstrap(sshSession, 900_000); await appendLog(previewId, `[PP] Bootstrap complete. Starting setup...\n`); if (repoConfig.aptPackages.length > 0) { diff --git a/backend/src/services/ec2.ts b/backend/src/services/ec2.ts index 9b174e78..9fa6d48c 100644 --- a/backend/src/services/ec2.ts +++ b/backend/src/services/ec2.ts @@ -115,7 +115,9 @@ export async function createPreviewSecurityGroup(ec2: EC2Client, groupName: stri } const BOOTSTRAP_SCRIPT = `#!/bin/bash -set -e +set -euo pipefail +exec > >(tee -a /var/log/pp-bootstrap.log) 2>&1 +trap 'touch /var/lib/pp-bootstrap-failed' ERR apt-get update -y apt-get install -y curl git unzip build-essential @@ -125,17 +127,6 @@ systemctl enable docker systemctl start docker apt-get install -y docker-compose-plugin -# Allow root SSH with key auth (sshd_config may default to prohibit-password or no) -sed -i 's/^#*PermitRootLogin.*/PermitRootLogin without-password/' /etc/ssh/sshd_config -# Copy ubuntu's authorized_keys to root so PP can SSH as root -mkdir -p /root/.ssh -chmod 700 /root/.ssh -if [ -f /home/ubuntu/.ssh/authorized_keys ]; then - cp /home/ubuntu/.ssh/authorized_keys /root/.ssh/authorized_keys - chmod 600 /root/.ssh/authorized_keys -fi -systemctl reload sshd || service ssh reload - # NVM + Node LTS (installed as root, available to root SSH sessions) export HOME=/root curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash @@ -144,8 +135,18 @@ source "$NVM_DIR/nvm.sh" nvm install --lts nvm alias default lts/* -# Signal that bootstrap is complete +# Do not permit PP's root SSH session until every bootstrap step succeeded. +# That prevents PP from seeing SSH as ready while cloud-init is still mutating +# the host and turns the completion marker into a reliable readiness signal. touch /var/lib/pp-bootstrap-done +sed -i 's/^#*PermitRootLogin.*/PermitRootLogin without-password/' /etc/ssh/sshd_config +mkdir -p /root/.ssh +chmod 700 /root/.ssh +if [ -f /home/ubuntu/.ssh/authorized_keys ]; then + cp /home/ubuntu/.ssh/authorized_keys /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys +fi +systemctl reload sshd || service ssh reload `; export async function launchInstance(opts: { @@ -179,12 +180,17 @@ export async function launchInstance(opts: { export async function waitForInstanceRunning(ec2: EC2Client, instanceId: string, maxWaitMs = 300_000): Promise { const start = Date.now(); while (Date.now() - start < maxWaitMs) { - const res = await ec2.send(new DescribeInstancesCommand({ - InstanceIds: [instanceId], - })); - const inst = res.Reservations?.[0]?.Instances?.[0]; - if (inst?.State?.Name === "running" && inst.PublicIpAddress) { - return inst.PublicIpAddress; + try { + const res = await ec2.send(new DescribeInstancesCommand({ + InstanceIds: [instanceId], + })); + const inst = res.Reservations?.[0]?.Instances?.[0]; + if (inst?.State?.Name === "running" && inst.PublicIpAddress) { + return inst.PublicIpAddress; + } + } catch (e: any) { + // AWS eventual consistency: instance may not be visible immediately after RunInstances + if (e.name !== "InvalidInstanceID.NotFound") throw e; } await sleep(5000); } diff --git a/backend/src/services/ssh.ts b/backend/src/services/ssh.ts index c2cb4fc6..3974484a 100644 --- a/backend/src/services/ssh.ts +++ b/backend/src/services/ssh.ts @@ -63,22 +63,32 @@ function tryConnect(host: string, privateKey: string, timeoutMs: number): Promis username: "root", privateKey, readyTimeout: timeoutMs, + keepaliveInterval: 10000, + keepaliveCountMax: 3, algorithms: { - serverHostKey: ["ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521"], + serverHostKey: ["ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", "ssh-ed25519"], }, }); }); } +const EXEC_TIMEOUT_MS = 30_000; + function execOnConn(conn: Client, command: string): Promise<{ stdout: string; stderr: string; code: number }> { return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`SSH exec timed out: ${command.slice(0, 60)}`)), EXEC_TIMEOUT_MS); + conn.exec(command, (err, stream) => { - if (err) return reject(err); + if (err) { + clearTimeout(timer); + return reject(err); + } let stdout = ""; let stderr = ""; stream.on("data", (d: Buffer) => { stdout += d.toString(); }); stream.stderr.on("data", (d: Buffer) => { stderr += d.toString(); }); stream.on("close", (code: number) => { + clearTimeout(timer); resolve({ stdout, stderr, code: code ?? 0 }); }); }); @@ -88,8 +98,12 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise { const start = Date.now(); while (Date.now() - start < maxWaitMs) { - const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting"); - if (res.stdout.trim() === "done") return; + try { + const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting"); + if (res.stdout.trim() === "done") return; + } catch { + // transient SSH exec error (e.g. timeout, channel reset) — retry + } await sleep(10_000); } throw new Error("EC2 bootstrap did not complete within timeout");