feat(deploy, ec2, ssh): add timeout parameters for SSH and Git commands, improve AWS credentials logging
Deploy / Build (push) Successful in 35s
Deploy / Build and Push Docker Image (push) Successful in 1m10s

This commit is contained in:
2026-07-26 15:14:16 +02:00
parent accbd0d2c6
commit d86b35f231
3 changed files with 34 additions and 22 deletions
+13 -5
View File
@@ -35,6 +35,10 @@ const APT_WAIT =
`fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 ` + `fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 ` +
`&& sleep 2 || break; done'`; `&& sleep 2 || break; done'`;
const SSH_STEP_TIMEOUT_MS = 20 * 60_000;
const GIT_STEP_TIMEOUT_MS = 10 * 60_000;
const PREINSTALL_TIMEOUT_MS = 20 * 60_000;
const activeSshSessions = new Map<number, SshSession>(); const activeSshSessions = new Map<number, SshSession>();
export function abortJobForPreview(previewId: number) { export function abortJobForPreview(previewId: number) {
@@ -324,7 +328,7 @@ async function firstDeploy(
const cloneCommand = giteaPat const cloneCommand = giteaPat
? `git -c http.extraHeader='Authorization: Basic ${authHeader}' clone '${cleanCloneUrl}' /opt/app` ? `git -c http.extraHeader='Authorization: Basic ${authHeader}' clone '${cleanCloneUrl}' /opt/app`
: `git clone '${cleanCloneUrl}' /opt/app`; : `git clone '${cleanCloneUrl}' /opt/app`;
const cloneResult = await sshSession.exec(cloneCommand); const cloneResult = await sshSession.exec(cloneCommand, GIT_STEP_TIMEOUT_MS);
if (cloneResult.stdout) await appendLog(previewId, cloneResult.stdout); if (cloneResult.stdout) await appendLog(previewId, cloneResult.stdout);
if (cloneResult.stderr) { if (cloneResult.stderr) {
// Mask PAT in stderr output too // Mask PAT in stderr output too
@@ -560,7 +564,7 @@ async function ensurePreinstall(previewId: number, sshSession: SshSession, repoC
for (const pkg of normalizeAptPackages(repoConfig.aptPackages)) aptPackages.add(pkg); for (const pkg of normalizeAptPackages(repoConfig.aptPackages)) aptPackages.add(pkg);
if (aptPackages.size > 0) { if (aptPackages.size > 0) {
await runSshStep(previewId, sshSession, `${APT_WAIT}; ${apt} update && ${apt} install -y ${[...aptPackages].join(" ")}`); await runSshStep(previewId, sshSession, `${APT_WAIT}; ${apt} update && ${apt} install -y ${[...aptPackages].join(" ")}`, true, PREINSTALL_TIMEOUT_MS);
} }
if (tools.includes("docker")) { if (tools.includes("docker")) {
@@ -569,6 +573,8 @@ async function ensurePreinstall(previewId: number, sshSession: SshSession, repoC
sshSession, sshSession,
`${APT_WAIT}; if ! command -v docker >/dev/null 2>&1; then curl -fsSL https://get.docker.com | sudo sh; fi; ` + `${APT_WAIT}; if ! command -v docker >/dev/null 2>&1; then curl -fsSL https://get.docker.com | sudo sh; fi; ` +
`sudo systemctl enable docker && sudo systemctl start docker; ${APT_WAIT}; ${apt} update && ${apt} install -y docker-compose-plugin; sudo usermod -aG docker ubuntu`, `sudo systemctl enable docker && sudo systemctl start docker; ${APT_WAIT}; ${apt} update && ${apt} install -y docker-compose-plugin; sudo usermod -aG docker ubuntu`,
true,
PREINSTALL_TIMEOUT_MS,
); );
} }
@@ -583,6 +589,8 @@ async function ensurePreinstall(previewId: number, sshSession: SshSession, repoC
`if [ -f /opt/app/.nvmrc ]; then cd /opt/app && nvm install && nvm use; ` + `if [ -f /opt/app/.nvmrc ]; then cd /opt/app && nvm install && nvm use; ` +
`else nvm install ${shellQuote(nodeVersion)} && nvm alias default ${shellQuote(nodeVersion)} && nvm use default; fi`, `else nvm install ${shellQuote(nodeVersion)} && nvm alias default ${shellQuote(nodeVersion)} && nvm use default; fi`,
), ),
true,
PREINSTALL_TIMEOUT_MS,
); );
} }
} }
@@ -636,10 +644,10 @@ function sleep(ms: number) {
return new Promise(r => setTimeout(r, ms)); return new Promise(r => setTimeout(r, ms));
} }
async function runSshStep(previewId: number, sshSession: SshSession, command: string, throwOnFail = true) { async function runSshStep(previewId: number, sshSession: SshSession, command: string, throwOnFail = true, timeoutMs = SSH_STEP_TIMEOUT_MS) {
checkAbortSession(sshSession); checkAbortSession(sshSession);
await appendLog(previewId, `$ ${command}\n`); await appendLog(previewId, `$ ${command}\n`);
const res = await sshSession.exec(command); const res = await sshSession.exec(command, timeoutMs);
if (res.stdout) await appendLog(previewId, res.stdout); if (res.stdout) await appendLog(previewId, res.stdout);
if (res.stderr) await appendLog(previewId, res.stderr); if (res.stderr) await appendLog(previewId, res.stderr);
if (throwOnFail && res.code !== 0) { if (throwOnFail && res.code !== 0) {
@@ -662,7 +670,7 @@ async function runGitStep(
) { ) {
checkAbortSession(sshSession); checkAbortSession(sshSession);
await appendLog(previewId, `$ ${displayCommand}\n`); await appendLog(previewId, `$ ${displayCommand}\n`);
const res = await sshSession.exec(realCommand); const res = await sshSession.exec(realCommand, GIT_STEP_TIMEOUT_MS);
const mask = (s: string) => maskValues.reduce((acc, m) => (m ? acc.split(m).join("****") : acc), s); const mask = (s: string) => maskValues.reduce((acc, m) => (m ? acc.split(m).join("****") : acc), s);
if (res.stdout) await appendLog(previewId, mask(res.stdout)); if (res.stdout) await appendLog(previewId, mask(res.stdout));
if (res.stderr) await appendLog(previewId, mask(res.stderr)); if (res.stderr) await appendLog(previewId, mask(res.stderr));
+1 -7
View File
@@ -61,13 +61,7 @@ export async function validateAwsCredentials(user: User): Promise<{ success: boo
try { try {
const accessKeyId = user.awsAccessKeyId ? decrypt(user.awsAccessKeyId) : ""; const accessKeyId = user.awsAccessKeyId ? decrypt(user.awsAccessKeyId) : "";
const region = user.awsRegion ?? ""; const region = user.awsRegion ?? "";
log.info("validateAwsCredentials: signer inputs", { log.info({ accessKeyIdLen: accessKeyId.length, region }, "validateAwsCredentials: signer inputs");
accessKeyId: JSON.stringify(accessKeyId),
accessKeyIdLen: accessKeyId.length,
accessKeyIdCharCodes: [...accessKeyId].map((c) => c.charCodeAt(0)),
region: JSON.stringify(region),
regionCharCodes: [...region].map((c) => c.charCodeAt(0)),
});
const sts = makeStsClient(user); const sts = makeStsClient(user);
const res = await sts.send(new GetCallerIdentityCommand({})); const res = await sts.send(new GetCallerIdentityCommand({}));
return { success: true, arn: res.Arn }; return { success: true, arn: res.Arn };
+20 -10
View File
@@ -4,7 +4,7 @@ import { createLogger } from "../lib/logger";
const log = createLogger("SSH"); const log = createLogger("SSH");
export interface SshSession { export interface SshSession {
exec(command: string): Promise<{ stdout: string; stderr: string; code: number }>; exec(command: string, timeoutMs?: number): Promise<{ stdout: string; stderr: string; code: number }>;
// Run a long-lived command (e.g. `tail -f`) and receive its stdout/stderr as // Run a long-lived command (e.g. `tail -f`) and receive its stdout/stderr as
// it arrives. Unlike exec(), this never buffers or times out — the channel // it arrives. Unlike exec(), this never buffers or times out — the channel
// stays open until the returned handle's close() is called (or the connection // stays open until the returned handle's close() is called (or the connection
@@ -29,9 +29,9 @@ export async function connectSsh(host: string, privateKey: string, maxWaitMs = 3
aborted = true; aborted = true;
try { conn.end(); } catch {} try { conn.end(); } catch {}
}, },
async exec(command: string) { async exec(command: string, timeoutMs?: number) {
if (aborted) throw new Error("SSH session aborted"); if (aborted) throw new Error("SSH session aborted");
return execOnConn(conn, command); return execOnConn(conn, command, timeoutMs);
}, },
execStream(command: string, onData: (chunk: string) => void) { execStream(command: string, onData: (chunk: string) => void) {
let stream: any = null; let stream: any = null;
@@ -99,22 +99,32 @@ function tryConnect(host: string, privateKey: string, timeoutMs: number): Promis
const EXEC_TIMEOUT_MS = 30_000; const EXEC_TIMEOUT_MS = 30_000;
function execOnConn(conn: Client, command: string): Promise<{ stdout: string; stderr: string; code: number }> { function execOnConn(conn: Client, command: string, timeoutMs = EXEC_TIMEOUT_MS): Promise<{ stdout: string; stderr: string; code: number }> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(`SSH exec timed out: ${command.slice(0, 60)}`)), EXEC_TIMEOUT_MS); let stream: any = null;
let settled = false;
const settle = (fn: () => void) => {
if (settled) return;
settled = true;
clearTimeout(timer);
fn();
};
const timer = setTimeout(() => {
settle(() => reject(new Error(`SSH exec timed out after ${Math.round(timeoutMs / 1000)}s: ${command.slice(0, 60)}`)));
try { stream?.close(); } catch {}
}, timeoutMs);
conn.exec(command, (err, stream) => { conn.exec(command, (err, s) => {
if (err) { if (err) {
clearTimeout(timer); return settle(() => reject(err));
return reject(err);
} }
stream = s;
let stdout = ""; let stdout = "";
let stderr = ""; let stderr = "";
stream.on("data", (d: Buffer) => { stdout += d.toString(); }); stream.on("data", (d: Buffer) => { stdout += d.toString(); });
stream.stderr.on("data", (d: Buffer) => { stderr += d.toString(); }); stream.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
stream.on("close", (code: number) => { stream.on("close", (code: number) => {
clearTimeout(timer); settle(() => resolve({ stdout, stderr, code: code ?? 0 }));
resolve({ stdout, stderr, code: code ?? 0 });
}); });
}); });
}); });