fix(v2): critical bug fixes — root SSH, bootstrap wait, SG deletion order, PAT masking
- EC2 bootstrap now enables root SSH (copies authorized_keys to root, sets PermitRootLogin without-password, reloads sshd) so all commands run as root and NVM at /root/.nvm is accessible - Added /var/lib/pp-bootstrap-done sentinel; deploy waits for it before running any user commands — prevents race between SSH availability and user-data completion (docker/nvm install can take 3-5+ min) - Fixed stopPreview: terminate instance first, delete key pair next, then delete security group with 30s delay — SG deletion was previously attempted before termination causing it to fail - Fixed redeploy to always fetch fresh instanceIp/sshPrivateKey from DB rather than using potentially-stale preview parameter - Fixed .env writing to use base64 encoding via echo|base64-d to safely handle values with special characters, single quotes, and newlines - PAT and git clone URL now masked in preview logs (shows **** for password) - Fixed inactivity cron: removed dead inactivityMs variable, use join on repoConfig to avoid N+1, deduplicate pending INACTIVITY_STOP jobs - Fixed IAM policy UI: ec2:CreateKeyPair (backend uses CreateKeyPair, not ImportKeyPair which is a different AWS operation) - Admin panel: added Edit button with username/password form for users - Privacy page: fetch and display admin contactEmail from settings - pnpm-workspace.yaml: fix allowBuilds→onlyBuiltDependencies for pnpm 9 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,7 @@ import {
|
||||
deleteKeyPairAws,
|
||||
deleteSecurityGroupAws,
|
||||
} from "./ec2";
|
||||
import { connectSsh, type SshSession } from "./ssh";
|
||||
import { connectSsh, waitForBootstrap, type SshSession } from "./ssh";
|
||||
import {
|
||||
buildPrCommentBody,
|
||||
postComment,
|
||||
@@ -70,29 +70,6 @@ async function updateStatus(previewId: number, status: Preview["status"], extra:
|
||||
await prisma.preview.update({ where: { id: previewId }, data: { status, ...extra } });
|
||||
}
|
||||
|
||||
async function updateGiteaComment(user: User, preview: Preview, repoConfig: RepoConfig, statusLine: string, lastLogLines?: string) {
|
||||
const body = buildPrCommentBody({
|
||||
owner: repoConfig.repoOwner,
|
||||
repo: repoConfig.repoName,
|
||||
prNumber: preview.prNumber,
|
||||
status: statusLine,
|
||||
commitSha: preview.commitSha,
|
||||
updatedAt: new Date(),
|
||||
ppBaseUrl: env.PP_BASE_URL,
|
||||
lastLogLines,
|
||||
instanceIp: preview.instanceIp ?? undefined,
|
||||
port: preview.port,
|
||||
});
|
||||
|
||||
if (preview.giteaCommentId) {
|
||||
try {
|
||||
await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, preview.giteaCommentId, body);
|
||||
} catch (e) {
|
||||
log.warn({ e }, "Failed to update Gitea comment");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function runDeploy(jobId: number) {
|
||||
const job = await prisma.job.findUnique({ where: { id: jobId }, include: { preview: { include: { repoConfig: { include: { user: true } } } } } });
|
||||
if (!job || !job.preview) {
|
||||
@@ -194,7 +171,7 @@ async function firstDeploy(
|
||||
status: "🟡 Provisioning EC2 instance...",
|
||||
commitSha, updatedAt: new Date(), ppBaseUrl: env.PP_BASE_URL,
|
||||
});
|
||||
let commentId = await postComment(user, repoConfig.repoOwner, repoConfig.repoName, prNumber, commentBody);
|
||||
const commentId = await postComment(user, repoConfig.repoOwner, repoConfig.repoName, prNumber, commentBody);
|
||||
await prisma.preview.update({ where: { id: previewId }, data: { giteaCommentId: commentId } });
|
||||
|
||||
checkAbort(previewId);
|
||||
@@ -227,8 +204,7 @@ async function firstDeploy(
|
||||
});
|
||||
|
||||
await prisma.preview.update({ where: { id: previewId }, data: { instanceId } });
|
||||
|
||||
await appendLog(previewId, `[PP] EC2 instance ${instanceId} launched. Waiting for it to be running...\n`);
|
||||
await appendLog(previewId, `[PP] EC2 instance ${instanceId} launched. Waiting for running state...\n`);
|
||||
|
||||
checkAbort(previewId);
|
||||
|
||||
@@ -243,16 +219,21 @@ async function firstDeploy(
|
||||
})
|
||||
);
|
||||
|
||||
await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH...\n`);
|
||||
await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH (as root)...\n`);
|
||||
|
||||
checkAbort(previewId);
|
||||
|
||||
// Connect as root. Ubuntu user-data copies authorized_keys to root and enables root SSH.
|
||||
const sshSession = await connectSsh(instanceIp, privateKey, 300_000);
|
||||
activeSshSessions.set(previewId, sshSession);
|
||||
|
||||
try {
|
||||
await appendLog(previewId, `[PP] SSH connected. Waiting for bootstrap to complete...\n`);
|
||||
await waitForBootstrap(sshSession, 600_000);
|
||||
await appendLog(previewId, `[PP] Bootstrap complete. Starting setup...\n`);
|
||||
|
||||
if (repoConfig.aptPackages.length > 0) {
|
||||
await runSshStep(previewId, sshSession, `sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ${repoConfig.aptPackages.join(" ")}`);
|
||||
await runSshStep(previewId, sshSession, `DEBIAN_FRONTEND=noninteractive apt-get install -y ${repoConfig.aptPackages.join(" ")}`);
|
||||
}
|
||||
|
||||
const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
|
||||
@@ -260,14 +241,25 @@ async function firstDeploy(
|
||||
parsedUrl.username = encodeURIComponent(user.giteaUsername || "");
|
||||
parsedUrl.password = encodeURIComponent(giteaPat);
|
||||
const authCloneUrl = parsedUrl.toString();
|
||||
await runSshStep(previewId, sshSession, `git clone '${authCloneUrl}' /opt/app`);
|
||||
|
||||
// Log a masked version so PAT is not exposed in preview logs
|
||||
const maskedUrl = `${parsedUrl.protocol}//${parsedUrl.username}:****@${parsedUrl.hostname}${parsedUrl.port ? ":" + parsedUrl.port : ""}${parsedUrl.pathname}`;
|
||||
await appendLog(previewId, `$ git clone '${maskedUrl}' /opt/app\n`);
|
||||
const cloneResult = await sshSession.exec(`git clone '${authCloneUrl}' /opt/app`);
|
||||
if (cloneResult.stdout) await appendLog(previewId, cloneResult.stdout);
|
||||
if (cloneResult.stderr) {
|
||||
// Mask PAT in stderr output too
|
||||
const maskedStderr = cloneResult.stderr.replace(encodeURIComponent(giteaPat), "****").replace(giteaPat, "****");
|
||||
await appendLog(previewId, maskedStderr);
|
||||
}
|
||||
if (cloneResult.code !== 0) throw new Error(`git clone failed with exit code ${cloneResult.code}`);
|
||||
|
||||
await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`);
|
||||
|
||||
await setupAndBuild(previewId, sshSession, repoConfig, preview, commitSha, true);
|
||||
await setupAndBuild(previewId, sshSession, repoConfig, commitSha, true);
|
||||
|
||||
await updateStatus(previewId, "RUNNING", { commitSha, instanceIp, port: repoConfig.port, lastActivityAt: new Date() });
|
||||
|
||||
const freshPreview = await prisma.preview.findUnique({ where: { id: previewId } });
|
||||
await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, commentId,
|
||||
buildPrCommentBody({
|
||||
owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
|
||||
@@ -291,16 +283,20 @@ async function redeploy(
|
||||
prTitle: string,
|
||||
) {
|
||||
const previewId = preview.id;
|
||||
const instanceIp = preview.instanceIp!;
|
||||
const privateKey = decrypt(preview.sshPrivateKey!);
|
||||
|
||||
const sshSession = await connectSsh(instanceIp, privateKey, 30_000);
|
||||
// Always use fresh data from DB for connection details
|
||||
const freshPreview = await prisma.preview.findUnique({ where: { id: previewId } });
|
||||
if (!freshPreview?.instanceIp || !freshPreview?.sshPrivateKey) {
|
||||
throw new Error("No active instance found for redeploy — cannot SSH in");
|
||||
}
|
||||
const instanceIp = freshPreview.instanceIp;
|
||||
const privateKey = decrypt(freshPreview.sshPrivateKey);
|
||||
|
||||
const sshSession = await connectSsh(instanceIp, privateKey, 60_000);
|
||||
activeSshSessions.set(previewId, sshSession);
|
||||
|
||||
await appendLog(previewId, `\n--- Redeploy: ${commitSha} ---\n`);
|
||||
|
||||
const freshPreview = await prisma.preview.findUnique({ where: { id: previewId } });
|
||||
|
||||
const commentBody = buildPrCommentBody({
|
||||
owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
|
||||
status: `🟡 Building... (EC2 at ${instanceIp})`,
|
||||
@@ -313,14 +309,14 @@ async function redeploy(
|
||||
if (repoConfig.useDockerCompose) {
|
||||
const composePath = repoConfig.composeFilePath || "docker-compose.yml";
|
||||
await runSshStep(previewId, sshSession, `cd /opt/app && docker compose -f ${composePath} down 2>&1 || true`);
|
||||
} else if (freshPreview?.pid) {
|
||||
} else if (freshPreview.pid) {
|
||||
await runSshStep(previewId, sshSession, `kill ${freshPreview.pid} 2>/dev/null || true; sleep 5; kill -9 ${freshPreview.pid} 2>/dev/null || true`);
|
||||
}
|
||||
|
||||
await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr && git reset --hard FETCH_HEAD`);
|
||||
|
||||
await updateStatus(previewId, "BUILDING");
|
||||
await setupAndBuild(previewId, sshSession, repoConfig, preview, commitSha, false);
|
||||
await setupAndBuild(previewId, sshSession, repoConfig, commitSha, false);
|
||||
|
||||
await updateStatus(previewId, "RUNNING", { commitSha, lastActivityAt: new Date() });
|
||||
|
||||
@@ -347,7 +343,6 @@ async function setupAndBuild(
|
||||
previewId: number,
|
||||
sshSession: SshSession,
|
||||
repoConfig: RepoConfig,
|
||||
preview: Preview,
|
||||
commitSha: string,
|
||||
isFirstProvision: boolean,
|
||||
) {
|
||||
@@ -355,14 +350,13 @@ async function setupAndBuild(
|
||||
const nvmCmd = `${NVM_PREFIX} if [ -f /opt/app/.nvmrc ]; then nvm install && nvm use; else nvm use default; fi`;
|
||||
await runSshStep(previewId, sshSession, `bash -c '${nvmCmd}'`, false);
|
||||
|
||||
// Write .env file
|
||||
// Write .env file safely using base64 to handle special chars and newlines
|
||||
const envVars = repoConfig.envVars as Record<string, string>;
|
||||
const envLines = Object.entries(envVars).map(([k, v]) => `${k}=${v}`).join("\\n");
|
||||
if (envLines) {
|
||||
await runSshStep(previewId, sshSession, `printf '${envLines}\\n' > /opt/app/.env`, false);
|
||||
} else {
|
||||
await runSshStep(previewId, sshSession, `touch /opt/app/.env`, false);
|
||||
}
|
||||
const envContent = Object.entries(envVars).map(([k, v]) => `${k}=${v}`).join("\n") + "\n";
|
||||
const envB64 = Buffer.from(envContent).toString("base64");
|
||||
await appendLog(previewId, `$ echo '<base64 .env> | base64 -d > /opt/app/.env'\n`);
|
||||
const envResult = await sshSession.exec(`echo '${envB64}' | base64 -d > /opt/app/.env`);
|
||||
if (envResult.code !== 0) throw new Error(".env write failed");
|
||||
|
||||
if (isFirstProvision) {
|
||||
for (const cmd of repoConfig.setupCommands) {
|
||||
@@ -436,15 +430,24 @@ export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED
|
||||
|
||||
if (preview.instanceId) {
|
||||
const ec2 = makeEc2Client(user);
|
||||
// Terminate instance first, then clean up key pair and security group
|
||||
try {
|
||||
await terminateInstance(ec2, preview.instanceId);
|
||||
} catch (e) {
|
||||
log.warn({ e, instanceId: preview.instanceId }, "Failed to terminate instance");
|
||||
}
|
||||
// Delete key pair immediately (doesn't depend on instance state)
|
||||
try {
|
||||
await deleteKeyPairAws(ec2, `pp-preview-${previewId}`);
|
||||
} catch {}
|
||||
try {
|
||||
await deleteSecurityGroupAws(ec2, `pp-preview-${previewId}`);
|
||||
} catch {}
|
||||
try {
|
||||
await terminateInstance(ec2, preview.instanceId);
|
||||
} catch {}
|
||||
// Delete security group after a delay to allow instance ENI detachment
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
await deleteSecurityGroupAws(ec2, `pp-preview-${previewId}`);
|
||||
} catch (e) {
|
||||
log.warn({ e, previewId }, "Failed to delete security group after termination");
|
||||
}
|
||||
}, 30_000);
|
||||
}
|
||||
|
||||
await prisma.preview.update({
|
||||
|
||||
@@ -118,15 +118,34 @@ const BOOTSTRAP_SCRIPT = `#!/bin/bash
|
||||
set -e
|
||||
apt-get update -y
|
||||
apt-get install -y curl git unzip build-essential
|
||||
|
||||
# Docker
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
systemctl enable docker
|
||||
systemctl start docker
|
||||
apt-get install -y docker-compose-plugin
|
||||
|
||||
# Allow root SSH with key auth (sshd_config may default to prohibit-password or no)
|
||||
sed -i 's/^#*PermitRootLogin.*/PermitRootLogin without-password/' /etc/ssh/sshd_config
|
||||
# Copy ubuntu's authorized_keys to root so PP can SSH as root
|
||||
mkdir -p /root/.ssh
|
||||
chmod 700 /root/.ssh
|
||||
if [ -f /home/ubuntu/.ssh/authorized_keys ]; then
|
||||
cp /home/ubuntu/.ssh/authorized_keys /root/.ssh/authorized_keys
|
||||
chmod 600 /root/.ssh/authorized_keys
|
||||
fi
|
||||
systemctl reload sshd || service ssh reload
|
||||
|
||||
# NVM + Node LTS (installed as root, available to root SSH sessions)
|
||||
export HOME=/root
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
|
||||
export NVM_DIR="/root/.nvm"
|
||||
source "$NVM_DIR/nvm.sh"
|
||||
nvm install --lts
|
||||
nvm alias default lts/*
|
||||
|
||||
# Signal that bootstrap is complete
|
||||
touch /var/lib/pp-bootstrap-done
|
||||
`;
|
||||
|
||||
export async function launchInstance(opts: {
|
||||
|
||||
@@ -60,7 +60,7 @@ function tryConnect(host: string, privateKey: string, timeoutMs: number): Promis
|
||||
conn.connect({
|
||||
host,
|
||||
port: 22,
|
||||
username: "ubuntu",
|
||||
username: "root",
|
||||
privateKey,
|
||||
readyTimeout: timeoutMs,
|
||||
algorithms: {
|
||||
@@ -85,6 +85,16 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st
|
||||
});
|
||||
}
|
||||
|
||||
export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < maxWaitMs) {
|
||||
const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
|
||||
if (res.stdout.trim() === "done") return;
|
||||
await sleep(10_000);
|
||||
}
|
||||
throw new Error("EC2 bootstrap did not complete within timeout");
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user