diff --git a/backend/src/services/deploy.ts b/backend/src/services/deploy.ts index 49439795..08a63dff 100644 --- a/backend/src/services/deploy.ts +++ b/backend/src/services/deploy.ts @@ -35,6 +35,10 @@ const APT_WAIT = `fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 ` + `&& sleep 2 || break; done'`; +const SSH_STEP_TIMEOUT_MS = 20 * 60_000; +const GIT_STEP_TIMEOUT_MS = 10 * 60_000; +const PREINSTALL_TIMEOUT_MS = 20 * 60_000; + const activeSshSessions = new Map(); export function abortJobForPreview(previewId: number) { @@ -324,7 +328,7 @@ async function firstDeploy( const cloneCommand = giteaPat ? `git -c http.extraHeader='Authorization: Basic ${authHeader}' clone '${cleanCloneUrl}' /opt/app` : `git clone '${cleanCloneUrl}' /opt/app`; - const cloneResult = await sshSession.exec(cloneCommand); + const cloneResult = await sshSession.exec(cloneCommand, GIT_STEP_TIMEOUT_MS); if (cloneResult.stdout) await appendLog(previewId, cloneResult.stdout); if (cloneResult.stderr) { // Mask PAT in stderr output too @@ -560,7 +564,7 @@ async function ensurePreinstall(previewId: number, sshSession: SshSession, repoC for (const pkg of normalizeAptPackages(repoConfig.aptPackages)) aptPackages.add(pkg); if (aptPackages.size > 0) { - await runSshStep(previewId, sshSession, `${APT_WAIT}; ${apt} update && ${apt} install -y ${[...aptPackages].join(" ")}`); + await runSshStep(previewId, sshSession, `${APT_WAIT}; ${apt} update && ${apt} install -y ${[...aptPackages].join(" ")}`, true, PREINSTALL_TIMEOUT_MS); } if (tools.includes("docker")) { @@ -569,6 +573,8 @@ async function ensurePreinstall(previewId: number, sshSession: SshSession, repoC sshSession, `${APT_WAIT}; if ! command -v docker >/dev/null 2>&1; then curl -fsSL https://get.docker.com | sudo sh; fi; ` + `sudo systemctl enable docker && sudo systemctl start docker; ${APT_WAIT}; ${apt} update && ${apt} install -y docker-compose-plugin; sudo usermod -aG docker ubuntu`, + true, + PREINSTALL_TIMEOUT_MS, ); } @@ -583,6 +589,8 @@ async function ensurePreinstall(previewId: number, sshSession: SshSession, repoC `if [ -f /opt/app/.nvmrc ]; then cd /opt/app && nvm install && nvm use; ` + `else nvm install ${shellQuote(nodeVersion)} && nvm alias default ${shellQuote(nodeVersion)} && nvm use default; fi`, ), + true, + PREINSTALL_TIMEOUT_MS, ); } } @@ -636,10 +644,10 @@ function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); } -async function runSshStep(previewId: number, sshSession: SshSession, command: string, throwOnFail = true) { +async function runSshStep(previewId: number, sshSession: SshSession, command: string, throwOnFail = true, timeoutMs = SSH_STEP_TIMEOUT_MS) { checkAbortSession(sshSession); await appendLog(previewId, `$ ${command}\n`); - const res = await sshSession.exec(command); + const res = await sshSession.exec(command, timeoutMs); if (res.stdout) await appendLog(previewId, res.stdout); if (res.stderr) await appendLog(previewId, res.stderr); if (throwOnFail && res.code !== 0) { @@ -662,7 +670,7 @@ async function runGitStep( ) { checkAbortSession(sshSession); await appendLog(previewId, `$ ${displayCommand}\n`); - const res = await sshSession.exec(realCommand); + const res = await sshSession.exec(realCommand, GIT_STEP_TIMEOUT_MS); const mask = (s: string) => maskValues.reduce((acc, m) => (m ? acc.split(m).join("****") : acc), s); if (res.stdout) await appendLog(previewId, mask(res.stdout)); if (res.stderr) await appendLog(previewId, mask(res.stderr)); diff --git a/backend/src/services/ec2.ts b/backend/src/services/ec2.ts index 2c0aca41..a065e818 100644 --- a/backend/src/services/ec2.ts +++ b/backend/src/services/ec2.ts @@ -61,13 +61,7 @@ export async function validateAwsCredentials(user: User): Promise<{ success: boo try { const accessKeyId = user.awsAccessKeyId ? decrypt(user.awsAccessKeyId) : ""; const region = user.awsRegion ?? ""; - log.info("validateAwsCredentials: signer inputs", { - accessKeyId: JSON.stringify(accessKeyId), - accessKeyIdLen: accessKeyId.length, - accessKeyIdCharCodes: [...accessKeyId].map((c) => c.charCodeAt(0)), - region: JSON.stringify(region), - regionCharCodes: [...region].map((c) => c.charCodeAt(0)), - }); + log.info({ accessKeyIdLen: accessKeyId.length, region }, "validateAwsCredentials: signer inputs"); const sts = makeStsClient(user); const res = await sts.send(new GetCallerIdentityCommand({})); return { success: true, arn: res.Arn }; diff --git a/backend/src/services/ssh.ts b/backend/src/services/ssh.ts index aa8fcaa7..4cc86c4d 100644 --- a/backend/src/services/ssh.ts +++ b/backend/src/services/ssh.ts @@ -4,7 +4,7 @@ import { createLogger } from "../lib/logger"; const log = createLogger("SSH"); export interface SshSession { - exec(command: string): Promise<{ stdout: string; stderr: string; code: number }>; + exec(command: string, timeoutMs?: number): Promise<{ stdout: string; stderr: string; code: number }>; // Run a long-lived command (e.g. `tail -f`) and receive its stdout/stderr as // it arrives. Unlike exec(), this never buffers or times out — the channel // stays open until the returned handle's close() is called (or the connection @@ -29,9 +29,9 @@ export async function connectSsh(host: string, privateKey: string, maxWaitMs = 3 aborted = true; try { conn.end(); } catch {} }, - async exec(command: string) { + async exec(command: string, timeoutMs?: number) { if (aborted) throw new Error("SSH session aborted"); - return execOnConn(conn, command); + return execOnConn(conn, command, timeoutMs); }, execStream(command: string, onData: (chunk: string) => void) { let stream: any = null; @@ -99,22 +99,32 @@ function tryConnect(host: string, privateKey: string, timeoutMs: number): Promis const EXEC_TIMEOUT_MS = 30_000; -function execOnConn(conn: Client, command: string): Promise<{ stdout: string; stderr: string; code: number }> { +function execOnConn(conn: Client, command: string, timeoutMs = EXEC_TIMEOUT_MS): Promise<{ stdout: string; stderr: string; code: number }> { return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(`SSH exec timed out: ${command.slice(0, 60)}`)), EXEC_TIMEOUT_MS); + let stream: any = null; + let settled = false; + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + fn(); + }; + const timer = setTimeout(() => { + settle(() => reject(new Error(`SSH exec timed out after ${Math.round(timeoutMs / 1000)}s: ${command.slice(0, 60)}`))); + try { stream?.close(); } catch {} + }, timeoutMs); - conn.exec(command, (err, stream) => { + conn.exec(command, (err, s) => { if (err) { - clearTimeout(timer); - return reject(err); + return settle(() => reject(err)); } + stream = s; let stdout = ""; let stderr = ""; stream.on("data", (d: Buffer) => { stdout += d.toString(); }); stream.stderr.on("data", (d: Buffer) => { stderr += d.toString(); }); stream.on("close", (code: number) => { - clearTimeout(timer); - resolve({ stdout, stderr, code: code ?? 0 }); + settle(() => resolve({ stdout, stderr, code: code ?? 0 })); }); }); });