fix(v2): critical bug fixes — root SSH, bootstrap wait, SG deletion order, PAT masking

- EC2 bootstrap now enables root SSH (copies authorized_keys to root, sets PermitRootLogin without-password, reloads sshd) so all commands run as root and NVM at /root/.nvm is accessible - Added /var/lib/pp-bootstrap-done sentinel; deploy waits for it before running any user commands — prevents race between SSH availability and user-data completion (docker/nvm install can take 3-5+ min) - Fixed stopPreview: terminate instance first, delete key pair next, then delete security group with 30s delay — SG deletion was previously attempted before termination causing it to fail - Fixed redeploy to always fetch fresh instanceIp/sshPrivateKey from DB rather than using potentially-stale preview parameter - Fixed .env writing to use base64 encoding via echo|base64-d to safely handle values with special characters, single quotes, and newlines - PAT and git clone URL now masked in preview logs (shows **** for password) - Fixed inactivity cron: removed dead inactivityMs variable, use join on repoConfig to avoid N+1, deduplicate pending INACTIVITY_STOP jobs - Fixed IAM policy UI: ec2:CreateKeyPair (backend uses CreateKeyPair, not ImportKeyPair which is a different AWS operation) - Admin panel: added Edit button with username/password form for users - Privacy page: fetch and display admin contactEmail from settings - pnpm-workspace.yaml: fix allowBuilds→onlyBuiltDependencies for pnpm 9 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-07-25 08:34:50 +00:00
parent eacbf59a10
commit 3f49511fb5
9 changed files with 211 additions and 81 deletions
@@ -13,7 +13,7 @@ import {
  deleteKeyPairAws,
  deleteSecurityGroupAws,
 } from "./ec2";
-import { connectSsh, type SshSession } from "./ssh";
+import { connectSsh, waitForBootstrap, type SshSession } from "./ssh";
 import {
  buildPrCommentBody,
  postComment,
@@ -70,29 +70,6 @@ async function updateStatus(previewId: number, status: Preview["status"], extra:
  await prisma.preview.update({ where: { id: previewId }, data: { status, ...extra } });
 }

-async function updateGiteaComment(user: User, preview: Preview, repoConfig: RepoConfig, statusLine: string, lastLogLines?: string) {
-  const body = buildPrCommentBody({
-    owner: repoConfig.repoOwner,
-    repo: repoConfig.repoName,
-    prNumber: preview.prNumber,
-    status: statusLine,
-    commitSha: preview.commitSha,
-    updatedAt: new Date(),
-    ppBaseUrl: env.PP_BASE_URL,
-    lastLogLines,
-    instanceIp: preview.instanceIp ?? undefined,
-    port: preview.port,
-  });
-
-  if (preview.giteaCommentId) {
-    try {
-      await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, preview.giteaCommentId, body);
-    } catch (e) {
-      log.warn({ e }, "Failed to update Gitea comment");
-    }
-  }
-}
-
 export async function runDeploy(jobId: number) {
  const job = await prisma.job.findUnique({ where: { id: jobId }, include: { preview: { include: { repoConfig: { include: { user: true } } } } } });
  if (!job || !job.preview) {
@@ -194,7 +171,7 @@ async function firstDeploy(
    status: "🟡 Provisioning EC2 instance...",
    commitSha, updatedAt: new Date(), ppBaseUrl: env.PP_BASE_URL,
  });
-  let commentId = await postComment(user, repoConfig.repoOwner, repoConfig.repoName, prNumber, commentBody);
+  const commentId = await postComment(user, repoConfig.repoOwner, repoConfig.repoName, prNumber, commentBody);
  await prisma.preview.update({ where: { id: previewId }, data: { giteaCommentId: commentId } });

  checkAbort(previewId);
@@ -227,8 +204,7 @@ async function firstDeploy(
  });

  await prisma.preview.update({ where: { id: previewId }, data: { instanceId } });
-
-  await appendLog(previewId, `[PP] EC2 instance ${instanceId} launched. Waiting for it to be running...\n`);
+  await appendLog(previewId, `[PP] EC2 instance ${instanceId} launched. Waiting for running state...\n`);

  checkAbort(previewId);

@@ -243,16 +219,21 @@ async function firstDeploy(
    })
  );

-  await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH...\n`);
+  await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH (as root)...\n`);

  checkAbort(previewId);

+  // Connect as root. Ubuntu user-data copies authorized_keys to root and enables root SSH.
  const sshSession = await connectSsh(instanceIp, privateKey, 300_000);
  activeSshSessions.set(previewId, sshSession);

  try {
+    await appendLog(previewId, `[PP] SSH connected. Waiting for bootstrap to complete...\n`);
+    await waitForBootstrap(sshSession, 600_000);
+    await appendLog(previewId, `[PP] Bootstrap complete. Starting setup...\n`);
+
    if (repoConfig.aptPackages.length > 0) {
-      await runSshStep(previewId, sshSession, `sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ${repoConfig.aptPackages.join(" ")}`);
+      await runSshStep(previewId, sshSession, `DEBIAN_FRONTEND=noninteractive apt-get install -y ${repoConfig.aptPackages.join(" ")}`);
    }

    const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
@@ -260,14 +241,25 @@ async function firstDeploy(
    parsedUrl.username = encodeURIComponent(user.giteaUsername || "");
    parsedUrl.password = encodeURIComponent(giteaPat);
    const authCloneUrl = parsedUrl.toString();
-    await runSshStep(previewId, sshSession, `git clone '${authCloneUrl}' /opt/app`);
+
+    // Log a masked version so PAT is not exposed in preview logs
+    const maskedUrl = `${parsedUrl.protocol}//${parsedUrl.username}:****@${parsedUrl.hostname}${parsedUrl.port ? ":" + parsedUrl.port : ""}${parsedUrl.pathname}`;
+    await appendLog(previewId, `$ git clone '${maskedUrl}' /opt/app\n`);
+    const cloneResult = await sshSession.exec(`git clone '${authCloneUrl}' /opt/app`);
+    if (cloneResult.stdout) await appendLog(previewId, cloneResult.stdout);
+    if (cloneResult.stderr) {
+      // Mask PAT in stderr output too
+      const maskedStderr = cloneResult.stderr.replace(encodeURIComponent(giteaPat), "****").replace(giteaPat, "****");
+      await appendLog(previewId, maskedStderr);
+    }
+    if (cloneResult.code !== 0) throw new Error(`git clone failed with exit code ${cloneResult.code}`);
+
    await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`);

-    await setupAndBuild(previewId, sshSession, repoConfig, preview, commitSha, true);
+    await setupAndBuild(previewId, sshSession, repoConfig, commitSha, true);

    await updateStatus(previewId, "RUNNING", { commitSha, instanceIp, port: repoConfig.port, lastActivityAt: new Date() });

-    const freshPreview = await prisma.preview.findUnique({ where: { id: previewId } });
    await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, commentId,
      buildPrCommentBody({
        owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
@@ -291,16 +283,20 @@ async function redeploy(
  prTitle: string,
 ) {
  const previewId = preview.id;
-  const instanceIp = preview.instanceIp!;
-  const privateKey = decrypt(preview.sshPrivateKey!);

-  const sshSession = await connectSsh(instanceIp, privateKey, 30_000);
+  // Always use fresh data from DB for connection details
+  const freshPreview = await prisma.preview.findUnique({ where: { id: previewId } });
+  if (!freshPreview?.instanceIp || !freshPreview?.sshPrivateKey) {
+    throw new Error("No active instance found for redeploy — cannot SSH in");
+  }
+  const instanceIp = freshPreview.instanceIp;
+  const privateKey = decrypt(freshPreview.sshPrivateKey);
+
+  const sshSession = await connectSsh(instanceIp, privateKey, 60_000);
  activeSshSessions.set(previewId, sshSession);

  await appendLog(previewId, `\n--- Redeploy: ${commitSha} ---\n`);

-  const freshPreview = await prisma.preview.findUnique({ where: { id: previewId } });
-
  const commentBody = buildPrCommentBody({
    owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
    status: `🟡 Building... (EC2 at ${instanceIp})`,
@@ -313,14 +309,14 @@ async function redeploy(
    if (repoConfig.useDockerCompose) {
      const composePath = repoConfig.composeFilePath || "docker-compose.yml";
      await runSshStep(previewId, sshSession, `cd /opt/app && docker compose -f ${composePath} down 2>&1 || true`);
-    } else if (freshPreview?.pid) {
+    } else if (freshPreview.pid) {
      await runSshStep(previewId, sshSession, `kill ${freshPreview.pid} 2>/dev/null || true; sleep 5; kill -9 ${freshPreview.pid} 2>/dev/null || true`);
    }

    await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr && git reset --hard FETCH_HEAD`);

    await updateStatus(previewId, "BUILDING");
-    await setupAndBuild(previewId, sshSession, repoConfig, preview, commitSha, false);
+    await setupAndBuild(previewId, sshSession, repoConfig, commitSha, false);

    await updateStatus(previewId, "RUNNING", { commitSha, lastActivityAt: new Date() });

@@ -347,7 +343,6 @@ async function setupAndBuild(
  previewId: number,
  sshSession: SshSession,
  repoConfig: RepoConfig,
-  preview: Preview,
  commitSha: string,
  isFirstProvision: boolean,
 ) {
@@ -355,14 +350,13 @@ async function setupAndBuild(
  const nvmCmd = `${NVM_PREFIX} if [ -f /opt/app/.nvmrc ]; then nvm install && nvm use; else nvm use default; fi`;
  await runSshStep(previewId, sshSession, `bash -c '${nvmCmd}'`, false);

-  // Write .env file
+  // Write .env file safely using base64 to handle special chars and newlines
  const envVars = repoConfig.envVars as Record<string, string>;
-  const envLines = Object.entries(envVars).map(([k, v]) => `${k}=${v}`).join("\\n");
-  if (envLines) {
-    await runSshStep(previewId, sshSession, `printf '${envLines}\\n' > /opt/app/.env`, false);
-  } else {
-    await runSshStep(previewId, sshSession, `touch /opt/app/.env`, false);
-  }
+  const envContent = Object.entries(envVars).map(([k, v]) => `${k}=${v}`).join("\n") + "\n";
+  const envB64 = Buffer.from(envContent).toString("base64");
+  await appendLog(previewId, `$ echo '<base64 .env> | base64 -d > /opt/app/.env'\n`);
+  const envResult = await sshSession.exec(`echo '${envB64}' | base64 -d > /opt/app/.env`);
+  if (envResult.code !== 0) throw new Error(".env write failed");

  if (isFirstProvision) {
    for (const cmd of repoConfig.setupCommands) {
@@ -436,15 +430,24 @@ export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED

  if (preview.instanceId) {
    const ec2 = makeEc2Client(user);
+    // Terminate instance first, then clean up key pair and security group
+    try {
+      await terminateInstance(ec2, preview.instanceId);
+    } catch (e) {
+      log.warn({ e, instanceId: preview.instanceId }, "Failed to terminate instance");
+    }
+    // Delete key pair immediately (doesn't depend on instance state)
    try {
      await deleteKeyPairAws(ec2, `pp-preview-${previewId}`);
    } catch {}
-    try {
-      await deleteSecurityGroupAws(ec2, `pp-preview-${previewId}`);
-    } catch {}
-    try {
-      await terminateInstance(ec2, preview.instanceId);
-    } catch {}
+    // Delete security group after a delay to allow instance ENI detachment
+    setTimeout(async () => {
+      try {
+        await deleteSecurityGroupAws(ec2, `pp-preview-${previewId}`);
+      } catch (e) {
+        log.warn({ e, previewId }, "Failed to delete security group after termination");
+      }
+    }, 30_000);
  }

  await prisma.preview.update({
@@ -118,15 +118,34 @@ const BOOTSTRAP_SCRIPT = `#!/bin/bash
 set -e
 apt-get update -y
 apt-get install -y curl git unzip build-essential
+
+# Docker
 curl -fsSL https://get.docker.com | sh
 systemctl enable docker
 systemctl start docker
 apt-get install -y docker-compose-plugin
+
+# Allow root SSH with key auth (sshd_config may default to prohibit-password or no)
+sed -i 's/^#*PermitRootLogin.*/PermitRootLogin without-password/' /etc/ssh/sshd_config
+# Copy ubuntu's authorized_keys to root so PP can SSH as root
+mkdir -p /root/.ssh
+chmod 700 /root/.ssh
+if [ -f /home/ubuntu/.ssh/authorized_keys ]; then
+  cp /home/ubuntu/.ssh/authorized_keys /root/.ssh/authorized_keys
+  chmod 600 /root/.ssh/authorized_keys
+fi
+systemctl reload sshd || service ssh reload
+
+# NVM + Node LTS (installed as root, available to root SSH sessions)
+export HOME=/root
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
 export NVM_DIR="/root/.nvm"
 source "$NVM_DIR/nvm.sh"
 nvm install --lts
 nvm alias default lts/*
+
+# Signal that bootstrap is complete
+touch /var/lib/pp-bootstrap-done
 `;

 export async function launchInstance(opts: {
@@ -60,7 +60,7 @@ function tryConnect(host: string, privateKey: string, timeoutMs: number): Promis
    conn.connect({
      host,
      port: 22,
-      username: "ubuntu",
+      username: "root",
      privateKey,
      readyTimeout: timeoutMs,
      algorithms: {
@@ -85,6 +85,16 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st
  });
 }

+export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < maxWaitMs) {
+    const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
+    if (res.stdout.trim() === "done") return;
+    await sleep(10_000);
+  }
+  throw new Error("EC2 bootstrap did not complete within timeout");
+}
+
 function sleep(ms: number) {
  return new Promise(r => setTimeout(r, ms));
 }
@@ -28,29 +28,32 @@ export function startCronWorkers() {
 }

 async function checkInactivity() {
-  const settings = await getAdminSettings();
  const now = new Date();

  const running = await prisma.preview.findMany({
    where: { status: "RUNNING" },
+    include: { repoConfig: { select: { inactivityHours: true } } },
  });

  for (const preview of running) {
-    const inactivityMs = settings.maxConcurrentInstancesPerUser; // will use actual inactivityHours from repoConfig
-    const repoConfig = await prisma.repoConfig.findUnique({ where: { id: preview.repoConfigId } });
-    if (!repoConfig) continue;
-
-    const deadline = new Date(preview.lastActivityAt.getTime() + repoConfig.inactivityHours * 3600 * 1000);
+    const inactivityHours = preview.repoConfig.inactivityHours;
+    const deadline = new Date(preview.lastActivityAt.getTime() + inactivityHours * 3600 * 1000);
    if (now >= deadline) {
-      log.info({ previewId: preview.id }, "Preview inactive, enqueuing INACTIVITY_STOP");
-      await prisma.job.create({
-        data: {
-          previewId: preview.id,
-          type: "INACTIVITY_STOP",
-          status: "PENDING",
-          payload: {},
-        },
+      log.info({ previewId: preview.id, inactivityHours }, "Preview inactive, enqueuing INACTIVITY_STOP");
+      // Only create one pending INACTIVITY_STOP per preview
+      const existingStop = await prisma.job.findFirst({
+        where: { previewId: preview.id, type: "INACTIVITY_STOP", status: { in: ["PENDING", "RUNNING"] } },
      });
+      if (!existingStop) {
+        await prisma.job.create({
+          data: {
+            previewId: preview.id,
+            type: "INACTIVITY_STOP",
+            status: "PENDING",
+            payload: {},
+          },
+        });
+      }
    }
  }
 }