Really huge mass update; Getting everything up-to-spec and implementing a wide range of features

backend/src/services/appLogs.ts

@@ -0,0 +1,133 @@
+import { prisma } from "../lib/db";
+import { decrypt } from "../lib/encryption";
+import { connectSsh, type SshSession } from "./ssh";
+import { createLogger } from "../lib/logger";
+
+const log = createLogger("APPLOGS");
+
+// The live app-log stream is deliberately separate from a preview's persisted
+// `logs` (which hold the PP deploy engine's own output). The running app's
+// stdout/stderr lives on the EC2 instance — in /opt/app/pp.log for a plain
+// process run, or in the container logs for docker-compose mode — so we tail it
+// on demand over an SSH channel while clients are watching. Nothing here is
+// written to the database; it's a passthrough to the WebSocket.
+//
+// A single tail per preview is shared (multiplexed) across all connected
+// viewers: the first subscriber opens the SSH connection, later ones attach to
+// it, and the connection is torn down once the last viewer disconnects. Recent
+// output is kept in a ring buffer so a late joiner sees history immediately
+// instead of a blank pane until the next line arrives.
+
+function tailCommand(useDockerCompose: boolean, composeFilePath: string | null): string {
+  if (useDockerCompose) {
+    const composePath = composeFilePath || "docker-compose.yml";
+    // `--tail=200` seeds recent history so the viewer isn't blank on connect;
+    // `-f` follows new output. Compose writes container logs to stdout/stderr,
+    // both of which execStream captures.
+    return `cd /opt/app && sudo docker compose -f '${composePath.replace(/'/g, `'"'"'`)}' logs -f --tail=200`;
+  }
+  // `-F` (not `-f`) follows by name and retries if the file is missing or gets
+  // rotated — the app may not have written pp.log yet when a client connects.
+  return `tail -n 200 -F /opt/app/pp.log 2>&1`;
+}
+
+const RING_BUFFER_BYTES = 64 * 1024;
+
+interface StreamEntry {
+  subscribers: Set<(chunk: string) => void>;
+  buffer: string[]; // recent chunks, replayed to late joiners
+  bufferBytes: number;
+  connecting: Promise<void>;
+  session: SshSession | null;
+  streamHandle: { close(): void } | null;
+  torndown: boolean;
+}
+
+const streams = new Map<number, StreamEntry>();
+
+function pushToBuffer(entry: StreamEntry, chunk: string) {
+  entry.buffer.push(chunk);
+  entry.bufferBytes += Buffer.byteLength(chunk, "utf8");
+  while (entry.bufferBytes > RING_BUFFER_BYTES && entry.buffer.length > 1) {
+    const dropped = entry.buffer.shift()!;
+    entry.bufferBytes -= Buffer.byteLength(dropped, "utf8");
+  }
+}
+
+function broadcast(entry: StreamEntry, chunk: string) {
+  pushToBuffer(entry, chunk);
+  for (const cb of entry.subscribers) {
+    try { cb(chunk); } catch {}
+  }
+}
+
+async function openStream(previewId: number, entry: StreamEntry) {
+  const preview = await prisma.preview.findUnique({
+    where: { id: previewId },
+    include: { repoConfig: true },
+  });
+
+  if (!preview || !preview.instanceIp || !preview.sshPrivateKey) {
+    broadcast(entry, "[app-logs] No running instance for this preview — app logs are only available while it is live.\n");
+    return;
+  }
+
+  const privateKey = decrypt(preview.sshPrivateKey);
+  const cmd = tailCommand(preview.repoConfig.useDockerCompose, preview.repoConfig.composeFilePath);
+
+  try {
+    const session = await connectSsh(preview.instanceIp, privateKey, 30_000);
+    // The last viewer may have left while we were connecting.
+    if (entry.torndown || entry.subscribers.size === 0) {
+      session.close();
+      return;
+    }
+    entry.session = session;
+    entry.streamHandle = session.execStream(cmd, (chunk) => broadcast(entry, chunk));
+  } catch (e: any) {
+    broadcast(entry, `[app-logs] Failed to connect to instance: ${e.message}\n`);
+    log.warn({ previewId, error: e.message }, "App log stream failed to connect");
+  }
+}
+
+function teardown(previewId: number, entry: StreamEntry) {
+  entry.torndown = true;
+  streams.delete(previewId);
+  try { entry.streamHandle?.close(); } catch {}
+  try { entry.session?.close(); } catch {}
+}
+
+// Subscribe to a preview's live app logs. The returned function unsubscribes;
+// when the last subscriber leaves, the shared SSH tail is torn down.
+export async function streamAppLogs(
+  previewId: number,
+  onData: (chunk: string) => void,
+): Promise<() => void> {
+  let entry = streams.get(previewId);
+
+  if (!entry) {
+    entry = {
+      subscribers: new Set(),
+      buffer: [],
+      bufferBytes: 0,
+      connecting: Promise.resolve(),
+      session: null,
+      streamHandle: null,
+      torndown: false,
+    };
+    streams.set(previewId, entry);
+    entry.connecting = openStream(previewId, entry);
+  }
+
+  entry.subscribers.add(onData);
+  // Replay buffered history so a late joiner isn't staring at a blank pane.
+  for (const chunk of entry.buffer) {
+    try { onData(chunk); } catch {}
+  }
+
+  const current = entry;
+  return () => {
+    current.subscribers.delete(onData);
+    if (current.subscribers.size === 0) teardown(previewId, current);
+  };
+}

backend/src/services/deploy.ts

@@ -10,10 +10,13 @@ import {
   launchInstance,
   waitForInstanceRunning,
   terminateInstance,
+  waitForInstanceTerminated,
   deleteKeyPairAws,
   deleteSecurityGroupAws,
 } from "./ec2";
-import { connectSsh, type SshSession } from "./ssh";
+import { connectSsh, waitForBootstrap, type SshSession } from "./ssh";
+import { computePreviewCostUsd, sessionCostUsd } from "../lib/cost";
+import { normalizeAptPackages, normalizePreinstallTools } from "../lib/preinstall";
 import {
   buildPrCommentBody,
   postComment,
@@ -23,6 +26,15 @@ import type { Preview, RepoConfig, User } from "@prisma/client";
 
 const log = createLogger("DEPLOY");
 
+// Block until Ubuntu's apt-daily / unattended-upgrades timers release every apt
+// lock, so a following `apt-get update`/`install` doesn't die with "Could not
+// get lock". Polls all four lock files for up to 300s (150 * 2s). `fuser`
+// returns 0 while a file is in use, non-zero once it's free.
+const APT_WAIT =
+  `sudo bash -c 'for i in $(seq 1 150); do ` +
+  `fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 ` +
+  `&& sleep 2 || break; done'`;
+
 const activeSshSessions = new Map<number, SshSession>();
 
 export function abortJobForPreview(previewId: number) {
@@ -33,12 +45,23 @@ export function abortJobForPreview(previewId: number) {
   }
 }
 
+// Prefix every non-empty line of a log chunk with a local-time `[HH:MM:SS]`
+// stamp. SSH execs buffer their whole output and hand it to appendLog at once,
+// so a command's lines share the stamp of the moment it completed — the useful
+// signal is *when each step happened*, which this captures. Blank lines (e.g.
+// the `\n---\n` separators) are left untouched so the log keeps its spacing.
+function stampLines(text: string): string {
+  const stamp = `[${new Date().toTimeString().slice(0, 8)}] `;
+  return text.replace(/^(?=.)/gm, stamp);
+}
+
 export async function appendLog(previewId: number, text: string) {
   const settings = await getAdminSettings();
   const preview = await prisma.preview.findUnique({ where: { id: previewId } });
   if (!preview) return;
 
-  let logs = (preview.logs || "") + text;
+  const stamped = stampLines(text);
+  let logs = (preview.logs || "") + stamped;
   if (Buffer.byteLength(logs, "utf8") > settings.logSizeLimitBytes) {
     const marker = "--- logs truncated ---\n";
     while (Buffer.byteLength(logs, "utf8") > settings.logSizeLimitBytes) {
@@ -51,7 +74,7 @@ export async function appendLog(previewId: number, text: string) {
 
   await prisma.preview.update({ where: { id: previewId }, data: { logs } });
 
-  broadcastLogUpdate(previewId, text);
+  broadcastLogUpdate(previewId, stamped);
 }
 
 const logSubscribers = new Map<number, Set<(text: string) => void>>();
@@ -94,7 +117,14 @@ export async function runDeploy(jobId: number) {
     if (isFirstDeploy) {
       await firstDeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle, cloneUrl);
     } else {
-      await redeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle);
+      const freshPreview = await prisma.preview.findUnique({ where: { id: preview.id } });
+      if (freshPreview?.instanceType && freshPreview.instanceType !== repoConfig.instanceType) {
+        await appendLog(preview.id, `[PP] Instance type changed from ${freshPreview.instanceType} to ${repoConfig.instanceType}; provisioning a fresh instance.\n`);
+        await stopPreview(preview.id, "STOPPED", "Instance type changed", true);
+        await firstDeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle, cloneUrl);
+      } else {
+        await redeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle);
+      }
     }
 
     await prisma.job.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: new Date() } });
@@ -168,7 +198,11 @@ async function firstDeploy(
     throw new Error("Concurrent instance limit reached");
   }
 
-  await updateStatus(previewId, "PROVISIONING", { commitSha, prNumber, prTitle });
+  // A restart from a STOPPED/IGNORED state reuses the same Preview row (see
+  // `/pp start` and stopped-branch push in webhook.ts), so its `logs` still hold
+  // the previous run's output. Clear them here so a fresh provision starts with a
+  // clean log instead of appending under the old EC2's build output.
+  await updateStatus(previewId, "PROVISIONING", { commitSha, prNumber, prTitle, logs: "", stopReason: null, stoppedAt: null });
 
   const commentBody = buildPrCommentBody({
     owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
@@ -207,7 +241,27 @@ async function firstDeploy(
     },
   });
 
-  await prisma.preview.update({ where: { id: previewId }, data: { instanceId } });
+  // Start a cost session for the new instance. If a prior session was left open
+  // (e.g. a failed deploy whose instance was never stopped before this restart),
+  // finalize its accrued cost first so it isn't lost when we overwrite the launch
+  // timestamp. See lib/cost.ts.
+  const priorCost = await prisma.preview.findUnique({
+    where: { id: previewId },
+    select: { accumulatedCostUsd: true, instanceLaunchedAt: true, instanceType: true },
+  });
+  let accumulatedCostUsd = priorCost?.accumulatedCostUsd ?? 0;
+  if (priorCost?.instanceLaunchedAt) {
+    accumulatedCostUsd += sessionCostUsd(priorCost.instanceLaunchedAt, new Date(), priorCost.instanceType);
+  }
+  await prisma.preview.update({
+    where: { id: previewId },
+    data: {
+      instanceId,
+      instanceType: repoConfig.instanceType,
+      instanceLaunchedAt: new Date(),
+      accumulatedCostUsd,
+    },
+  });
   await appendLog(previewId, `[PP] EC2 instance ${instanceId} launched. Waiting for running state...\n`);
 
   checkAbort(previewId);
@@ -223,25 +277,33 @@ async function firstDeploy(
     })
   );
 
-  await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH (as root)...\n`);
+  await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH (as ubuntu)...\n`);
 
   checkAbort(previewId);
 
-  // Ubuntu cloud images support command execution through their provisioned user.
-  const sshSession = await connectSsh(instanceIp, privateKey, 300_000);
+  // The `ubuntu` user accepts SSH almost immediately at boot, but cloud-init is
+  // still disabling apt timers and installing PP's base tools. Open a session
+  // just to wait for its completion marker before touching the instance,
+  // otherwise we can still hit apt-lock races below.
+  const bootstrapSession = await connectSsh(instanceIp, privateKey, 300_000);
+  activeSshSessions.set(previewId, bootstrapSession);
+  await appendLog(previewId, `[PP] Waiting for instance bootstrap to finish...\n`);
+  try {
+    checkAbort(previewId);
+    await waitForBootstrap(bootstrapSession);
+  } finally {
+    bootstrapSession.close();
+  }
+  checkAbort(previewId);
+
+  // Reconnect for the actual work so the marker-wait session stays short-lived
+  // and cannot carry stale state into the deploy steps.
+  const sshSession = await connectSsh(instanceIp, privateKey, 60_000);
   activeSshSessions.set(previewId, sshSession);
 
   try {
-    // Root SSH is enabled as the final bootstrap step, after the completion
-    // marker is written. A successful root connection is therefore the
-    // readiness signal; polling the same long-lived SSH connection can leave
-    // a completed instance stuck when its exec channel goes stale.
     await appendLog(previewId, `[PP] Bootstrap complete. Starting setup...\n`);
 
-    if (repoConfig.aptPackages.length > 0) {
-      await runSshStep(previewId, sshSession, `sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ${repoConfig.aptPackages.join(" ")}`);
-    }
-
     const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
     const parsedUrl = new URL(cloneUrl.startsWith("http") ? cloneUrl : `https://${cloneUrl}`);
     parsedUrl.username = "";
@@ -249,6 +311,12 @@ async function firstDeploy(
     const cleanCloneUrl = parsedUrl.toString();
     const authHeader = Buffer.from(`${user.giteaUsername || ""}:${giteaPat}`).toString("base64");
 
+    // We SSH in as the unprivileged `ubuntu` user, but /opt is root-owned, so
+    // an unprivileged clone into /opt/app fails with EACCES. Create the target
+    // directory up front and hand it to ubuntu so the clone (and every later
+    // step) can write without sudo.
+    await runSshStep(previewId, sshSession, `sudo mkdir -p /opt/app && sudo chown ubuntu:ubuntu /opt/app`);
+
     // Keep the PAT out of both preview logs and the URL. Gitea deployments can
     // reject userinfo URLs, while Git's per-command HTTP header works for both
     // private repositories and reverse proxies.
@@ -265,11 +333,21 @@ async function firstDeploy(
     }
     if (cloneResult.code !== 0) throw new Error(`git clone failed with exit code ${cloneResult.code}`);
 
-    await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`);
+    // The PR ref lives in the same private repo, so the fetch needs the same
+    // auth as the clone — otherwise git prompts for a username and aborts.
+    const authOption = giteaAuthOption(authHeader, giteaPat);
+    await runGitStep(
+      previewId,
+      sshSession,
+      `cd /opt/app && git ${authOption}fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`,
+      `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`,
+      authHeader,
+      giteaPat,
+    );
 
     await setupAndBuild(previewId, sshSession, repoConfig, commitSha, true);
 
-    await updateStatus(previewId, "RUNNING", { commitSha, instanceIp, port: repoConfig.port, lastActivityAt: new Date() });
+    await updateStatus(previewId, "RUNNING", { commitSha, instanceIp, port: repoConfig.port, lastActivityAt: new Date(), stopReason: null, stoppedAt: null });
 
     await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, commentId,
       buildPrCommentBody({
@@ -319,17 +397,45 @@ async function redeploy(
   try {
     if (repoConfig.useDockerCompose) {
       const composePath = repoConfig.composeFilePath || "docker-compose.yml";
-      await runSshStep(previewId, sshSession, `cd /opt/app && docker compose -f ${composePath} down 2>&1 || true`);
-    } else if (freshPreview.pid) {
-      await runSshStep(previewId, sshSession, `kill ${freshPreview.pid} 2>/dev/null || true; sleep 5; kill -9 ${freshPreview.pid} 2>/dev/null || true`);
+      await runSshStep(previewId, sshSession, `cd /opt/app && sudo docker compose -f ${shellQuote(composePath)} down 2>&1 || true`);
+    } else {
+      // Non-compose: stop the previous process. `pnpm start` (and npm/yarn) run
+      // the real server as a CHILD of the recorded PID, so killing just that PID
+      // can orphan the server on the app port; and a prior failed/timed-out
+      // deploy may not have recorded a PID at all. In both cases the next start
+      // dies with EADDRINUSE. Kill the recorded PID if we have one, then free the
+      // port itself as the authoritative backstop — `fuser -k` kills whatever is
+      // actually listening, clearing orphans regardless of how they were spawned.
+      const killPid = freshPreview.pid
+        ? `kill ${freshPreview.pid} 2>/dev/null || true; sleep 3; kill -9 ${freshPreview.pid} 2>/dev/null || true; `
+        : "";
+      await runSshStep(previewId, sshSession, `bash -c '${killPid}fuser -k ${repoConfig.port}/tcp 2>/dev/null || true; sleep 1; true'`);
     }
 
-    await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr && git reset --hard FETCH_HEAD`);
+    // Reuse the same per-command auth as firstDeploy; the checked-out repo has
+    // no persisted credentials, so every fetch must supply the header itself.
+    const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
+    const authHeader = Buffer.from(`${user.giteaUsername || ""}:${giteaPat}`).toString("base64");
+    const authOption = giteaAuthOption(authHeader, giteaPat);
+    // The working tree is already on the `pp-pr` branch from the first deploy,
+    // and git refuses to fetch directly into a checked-out branch ref of a
+    // non-bare repo ("Refusing to fetch into current branch"). Fetch into
+    // FETCH_HEAD instead (no branch-ref update), then force `pp-pr` to it with
+    // `checkout -B`, which also updates the working tree — equivalent to the old
+    // fetch+checkout+reset but without touching the live branch ref.
+    await runGitStep(
+      previewId,
+      sshSession,
+      `cd /opt/app && git ${authOption}fetch origin pull/${prNumber}/head && git checkout -B pp-pr FETCH_HEAD`,
+      `cd /opt/app && git fetch origin pull/${prNumber}/head && git checkout -B pp-pr FETCH_HEAD`,
+      authHeader,
+      giteaPat,
+    );
 
     await updateStatus(previewId, "BUILDING");
     await setupAndBuild(previewId, sshSession, repoConfig, commitSha, false);
 
-    await updateStatus(previewId, "RUNNING", { commitSha, lastActivityAt: new Date() });
+    await updateStatus(previewId, "RUNNING", { commitSha, lastActivityAt: new Date(), stopReason: null, stoppedAt: null });
 
     await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, newCommentId,
       buildPrCommentBody({
@@ -346,8 +452,21 @@ async function redeploy(
 
 const NVM_PREFIX = `export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh" 2>/dev/null;`;
 
-function withNvm(cmd: string): string {
-  return `bash -c '${NVM_PREFIX} ${cmd.replace(/'/g, `'"'"'`)}'`;
+function shellQuote(value: string): string {
+  return `'${value.replace(/'/g, `'"'"'`)}'`;
+}
+
+function bashLc(cmd: string): string {
+  return `bash -lc ${shellQuote(cmd)}`;
+}
+
+function usesNode(repoConfig: RepoConfig): boolean {
+  return normalizePreinstallTools((repoConfig as any).preinstallTools, repoConfig.useDockerCompose).includes("node");
+}
+
+function withRuntime(repoConfig: RepoConfig, cmd: string): string {
+  const prefix = usesNode(repoConfig) ? `${NVM_PREFIX} ` : "";
+  return bashLc(`${prefix}cd /opt/app && ${cmd}`);
 }
 
 async function setupAndBuild(
@@ -357,9 +476,7 @@ async function setupAndBuild(
   commitSha: string,
   isFirstProvision: boolean,
 ) {
-  // Detect and use Node version
-  const nvmCmd = `${NVM_PREFIX} if [ -f /opt/app/.nvmrc ]; then nvm install && nvm use; else nvm use default; fi`;
-  await runSshStep(previewId, sshSession, `bash -c '${nvmCmd}'`, false);
+  await ensurePreinstall(previewId, sshSession, repoConfig);
 
   // Write .env file safely using base64 to handle special chars and newlines
   const envVars = repoConfig.envVars as Record<string, string>;
@@ -371,7 +488,7 @@ async function setupAndBuild(
 
   if (isFirstProvision) {
     for (const cmd of repoConfig.setupCommands) {
-      await runSshStep(previewId, sshSession, withNvm(`cd /opt/app && ${cmd}`));
+      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
     }
   }
 
@@ -379,26 +496,146 @@ async function setupAndBuild(
 
   if (repoConfig.useDockerCompose) {
     const composePath = repoConfig.composeFilePath || "docker-compose.yml";
-    await runSshStep(previewId, sshSession, `cd /opt/app && docker compose -f ${composePath} up -d --build --force-recreate 2>&1`);
+    await runSshStep(previewId, sshSession, `cd /opt/app && sudo docker compose -f ${shellQuote(composePath)} up -d --build --force-recreate 2>&1`);
   } else {
+    // We SSH in and build as `ubuntu`, but a prior Docker-mode run (or a compose
+    // bind mount) can leave root-owned files under /opt/app — chiefly
+    // dependency folders — which makes package managers fail with EACCES when
+    // they try to rewrite them. Reclaim ownership before the build so switching
+    // a preview from docker-compose to a plain process doesn't wedge on
+    // permissions.
+    await runSshStep(previewId, sshSession, `sudo chown -R ubuntu:ubuntu /opt/app`);
+
     for (const cmd of repoConfig.buildCommands) {
-      await runSshStep(previewId, sshSession, withNvm(`cd /opt/app && ${cmd}`));
+      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
     }
     for (const cmd of repoConfig.postBuildCommands) {
-      await runSshStep(previewId, sshSession, withNvm(`cd /opt/app && ${cmd}`));
+      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
     }
 
-    if (repoConfig.runCommand) {
-      const startCmd = withNvm(`cd /opt/app && nohup ${repoConfig.runCommand} > /opt/app/pp.log 2>&1 & echo $!`);
-      const res = await runSshStep(previewId, sshSession, startCmd);
-      const pid = parseInt(res.stdout.trim(), 10);
-      if (!isNaN(pid)) {
-        await prisma.preview.update({ where: { id: previewId }, data: { pid } });
-      }
+    // Non-compose mode has no start step other than `runCommand`. If it's blank
+    // there is nothing to launch — fail loudly instead of marking the preview
+    // RUNNING with no process behind it (a misleading green preview).
+    const runCommand = (repoConfig.runCommand || "").trim();
+    if (!runCommand) {
+      throw new Error(
+        "No run command is configured for this repo (non-compose mode), so there is nothing to start. " +
+          'Set a Run Command in the repo config (e.g. "pnpm start"), or enable Docker Compose mode.',
+      );
+    }
+
+    // Detaching a long-lived server over SSH is subtle. A plain `nohup <cmd> &`
+    // leaves the process in the exec channel's process group, and sshd keeps the
+    // channel open until it exits — so the exec never returns and PP's 30s cap
+    // fires even though the app started fine. (Verified on a live instance: a
+    // bare `sleep` detaches instantly, but any long-running node/pnpm process
+    // hangs the channel.) `disown` removes the job from the shell so the channel
+    // can close immediately; `nohup` keeps it running past our session teardown;
+    // `< /dev/null` frees stdin. `echo $!` still yields the launched PID.
+    const startCmd = withRuntime(repoConfig, `{ nohup ${runCommand} > /opt/app/pp.log 2>&1 < /dev/null & } ; disown ; echo $!`);
+    const res = await runSshStep(previewId, sshSession, startCmd);
+    const pid = parseInt(res.stdout.trim(), 10);
+    if (!isNaN(pid)) {
+      await prisma.preview.update({ where: { id: previewId }, data: { pid } });
+      await healthCheck(previewId, sshSession, pid, repoConfig.port);
     }
   }
 }
 
+async function ensurePreinstall(previewId: number, sshSession: SshSession, repoConfig: RepoConfig) {
+  const tools = normalizePreinstallTools((repoConfig as any).preinstallTools, repoConfig.useDockerCompose);
+  if (tools.length === 0 && repoConfig.aptPackages.length === 0) {
+    await appendLog(previewId, "[PP] No preinstall options selected.\n");
+    return;
+  }
+
+  await appendLog(previewId, `[PP] Installing selected preinstall options: ${tools.length ? tools.join(", ") : "custom apt packages only"}.\n`);
+
+  const apt = `sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300`;
+  const aptPackages = new Set<string>();
+  if (tools.includes("python")) ["python3", "python3-pip", "python3-venv"].forEach(p => aptPackages.add(p));
+  if (tools.includes("go")) aptPackages.add("golang-go");
+  if (tools.includes("lua")) ["lua5.4", "luarocks"].forEach(p => aptPackages.add(p));
+  if (tools.includes("build-essential")) ["build-essential", "pkg-config"].forEach(p => aptPackages.add(p));
+  for (const pkg of normalizeAptPackages(repoConfig.aptPackages)) aptPackages.add(pkg);
+
+  if (aptPackages.size > 0) {
+    await runSshStep(previewId, sshSession, `${APT_WAIT}; ${apt} update && ${apt} install -y ${[...aptPackages].join(" ")}`);
+  }
+
+  if (tools.includes("docker")) {
+    await runSshStep(
+      previewId,
+      sshSession,
+      `${APT_WAIT}; if ! command -v docker >/dev/null 2>&1; then curl -fsSL https://get.docker.com | sudo sh; fi; ` +
+        `sudo systemctl enable docker && sudo systemctl start docker; ${APT_WAIT}; ${apt} update && ${apt} install -y docker-compose-plugin; sudo usermod -aG docker ubuntu`,
+    );
+  }
+
+  if (tools.includes("node")) {
+    const nodeVersion = String((repoConfig as any).nodeVersion || "lts/*");
+    await runSshStep(
+      previewId,
+      sshSession,
+      bashLc(
+        `if [ ! -s "$HOME/.nvm/nvm.sh" ]; then curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash; fi; ` +
+          `export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh"; ` +
+          `if [ -f /opt/app/.nvmrc ]; then cd /opt/app && nvm install && nvm use; ` +
+          `else nvm install ${shellQuote(nodeVersion)} && nvm alias default ${shellQuote(nodeVersion)} && nvm use default; fi`,
+      ),
+    );
+  }
+}
+
+// Wait for the freshly-launched app to come up. `nohup ... &` detaches the
+// process and routes its output to pp.log, so nothing from the server reaches
+// the preview logs — a clean start and an instant crash look identical. Poll
+// until either the process dies or *anything* accepts a TCP connection on the
+// app port (a bare connect, not an HTTP 2xx — the app owns the port either way),
+// then surface the first lines of pp.log so the startup is visible. Polls from
+// Node in short bursts because a single SSH exec is capped at 30s (see ssh.ts),
+// so a long in-shell loop would trip that timeout.
+const HEALTH_TIMEOUT_MS = 60_000;
+const HEALTH_INTERVAL_MS = 2_000;
+
+async function healthCheck(previewId: number, sshSession: SshSession, pid: number, port: number) {
+  await appendLog(previewId, `[PP] Waiting up to ${HEALTH_TIMEOUT_MS / 1000}s for the app to respond on port ${port}...\n`);
+
+  const start = Date.now();
+  let healthy = false;
+  let exited = false;
+
+  while (Date.now() - start < HEALTH_TIMEOUT_MS) {
+    checkAbortSession(sshSession);
+    // exit 2 = process gone, 0 = port accepted a connection, 1 = not yet.
+    // `/dev/tcp` is a bash builtin, so no nc/curl dependency; the connect runs
+    // in a subshell so its fd can't leak into later commands on this channel.
+    const probe = await sshSession.exec(
+      `bash -c 'kill -0 ${pid} 2>/dev/null || exit 2; (exec 3<>/dev/tcp/127.0.0.1/${port}) 2>/dev/null && exit 0 || exit 1'`,
+    );
+    if (probe.code === 0) { healthy = true; break; }
+    if (probe.code === 2) { exited = true; break; }
+    await sleep(HEALTH_INTERVAL_MS);
+  }
+
+  const tail = await sshSession.exec(`tail -n 40 /opt/app/pp.log 2>/dev/null`);
+  await appendLog(previewId, `--- server startup output (pp.log) ---\n`);
+  if (tail.stdout) await appendLog(previewId, tail.stdout);
+  await appendLog(previewId, `\n---\n`);
+
+  if (healthy) {
+    await appendLog(previewId, `[PP] Health check passed — port ${port} is accepting connections (pid ${pid}).\n`);
+  } else if (exited) {
+    throw new Error(`Server process ${pid} exited during startup — see pp.log output above.`);
+  } else {
+    throw new Error(`Health check timed out — nothing responded on port ${port} within ${HEALTH_TIMEOUT_MS / 1000}s. See pp.log output above.`);
+  }
+}
+
+function sleep(ms: number) {
+  return new Promise(r => setTimeout(r, ms));
+}
+
 async function runSshStep(previewId: number, sshSession: SshSession, command: string, throwOnFail = true) {
   checkAbortSession(sshSession);
   await appendLog(previewId, `$ ${command}\n`);
@@ -411,6 +648,37 @@ async function runSshStep(previewId: number, sshSession: SshSession, command: st
   return res;
 }
 
+// Like runSshStep, but for git commands that must carry the Gitea auth header.
+// The real command embeds the base64 credential inline (per-command, never
+// persisted to .git/config); `displayCommand` is the credential-free form we
+// echo to the logs, and any secret leaking into stdout/stderr is masked. This
+// mirrors the masking the initial clone already does.
+async function runGitStep(
+  previewId: number,
+  sshSession: SshSession,
+  realCommand: string,
+  displayCommand: string,
+  ...maskValues: string[]
+) {
+  checkAbortSession(sshSession);
+  await appendLog(previewId, `$ ${displayCommand}\n`);
+  const res = await sshSession.exec(realCommand);
+  const mask = (s: string) => maskValues.reduce((acc, m) => (m ? acc.split(m).join("****") : acc), s);
+  if (res.stdout) await appendLog(previewId, mask(res.stdout));
+  if (res.stderr) await appendLog(previewId, mask(res.stderr));
+  if (res.code !== 0) {
+    throw new Error(`Command failed with exit code ${res.code}: ${displayCommand}`);
+  }
+  return res;
+}
+
+// Builds the `-c http.extraHeader=...` git option that authenticates a single
+// git invocation against a private Gitea repo. Empty when there is no PAT
+// (public repos clone/fetch anonymously).
+function giteaAuthOption(authHeader: string, giteaPat: string): string {
+  return giteaPat ? `-c http.extraHeader='Authorization: Basic ${authHeader}' ` : "";
+}
+
 function checkAbortSession(session: SshSession) {
   if (session.aborted) throw new Error("ABORTED");
 }
@@ -429,7 +697,7 @@ function checkAbort(previewId: number) {
   }
 }
 
-export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED" = "STOPPED") {
+export async function stopPreview(previewId: number, status: "STOPPED" | "FAILED" = "STOPPED", reason = "Stopped", waitForCleanup = false) {
   const preview = await prisma.preview.findUnique({
     where: { id: previewId },
     include: { repoConfig: { include: { user: true } } },
@@ -441,34 +709,57 @@ export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED
 
   if (preview.instanceId) {
     const ec2 = makeEc2Client(user);
+    const instanceId = preview.instanceId;
     // Terminate instance first, then clean up key pair and security group
     try {
-      await terminateInstance(ec2, preview.instanceId);
+      await terminateInstance(ec2, instanceId);
     } catch (e) {
-      log.warn({ e, instanceId: preview.instanceId }, "Failed to terminate instance");
+      log.warn({ e, instanceId }, "Failed to terminate instance");
     }
     // Delete key pair immediately (doesn't depend on instance state)
     try {
       await deleteKeyPairAws(ec2, `pp-preview-${previewId}`);
     } catch {}
-    // Delete security group after a delay to allow instance ENI detachment
-    setTimeout(async () => {
+    // The security group can't be deleted until the instance's ENI is released,
+    // which only happens once the instance is fully terminated. Wait for that,
+    // then delete (deleteSecurityGroupAws also retries on DependencyViolation).
+    const cleanupSecurityGroup = async () => {
       try {
+        await waitForInstanceTerminated(ec2, instanceId);
         await deleteSecurityGroupAws(ec2, `pp-preview-${previewId}`);
       } catch (e) {
         log.warn({ e, previewId }, "Failed to delete security group after termination");
       }
-    }, 30_000);
+    };
+    if (waitForCleanup) await cleanupSecurityGroup();
+    else void cleanupSecurityGroup();
   }
 
+  // Close out the live cost session: fold the running instance's accrued cost
+  // into the finalized total and clear the launch timestamp so it stops billing.
+  const stoppedAt = new Date();
+  const finalCostUsd = computePreviewCostUsd(
+    {
+      accumulatedCostUsd: preview.accumulatedCostUsd,
+      instanceLaunchedAt: preview.instanceLaunchedAt,
+      instanceType: preview.instanceType,
+    },
+    stoppedAt,
+  );
+
+  const stopReason = reason.trim().slice(0, 128) || "Stopped";
+
   await prisma.preview.update({
     where: { id: previewId },
     data: {
-      status: reason,
-      stoppedAt: new Date(),
+      status,
+      stopReason,
+      stoppedAt,
       sshPrivateKey: null,
       sshKeyName: null,
       instanceId: null,
+      instanceLaunchedAt: null,
+      accumulatedCostUsd: finalCostUsd,
     },
   });
 
@@ -476,7 +767,7 @@ export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED
     owner: repoConfig.repoOwner,
     repo: repoConfig.repoName,
     prNumber: preview.prNumber,
-    status: "⚫ Stopped (inactivity timeout / PR closed / manual stop)",
+    status: `⚫ Stopped (${stopReason})`,
     commitSha: preview.commitSha,
     updatedAt: new Date(),
     ppBaseUrl: env.PP_BASE_URL,

backend/src/services/ec2.ts

@@ -59,6 +59,15 @@ export function makeStsClient(user: User): STSClient {
 
 export async function validateAwsCredentials(user: User): Promise<{ success: boolean; arn?: string; error?: string }> {
   try {
+    const accessKeyId = user.awsAccessKeyId ? decrypt(user.awsAccessKeyId) : "";
+    const region = user.awsRegion ?? "";
+    log.info("validateAwsCredentials: signer inputs", {
+      accessKeyId: JSON.stringify(accessKeyId),
+      accessKeyIdLen: accessKeyId.length,
+      accessKeyIdCharCodes: [...accessKeyId].map((c) => c.charCodeAt(0)),
+      region: JSON.stringify(region),
+      regionCharCodes: [...region].map((c) => c.charCodeAt(0)),
+    });
     const sts = makeStsClient(user);
     const res = await sts.send(new GetCallerIdentityCommand({}));
     return { success: true, arn: res.Arn };
@@ -118,20 +127,24 @@ const BOOTSTRAP_SCRIPT = `#!/bin/bash
 set -euo pipefail
 exec > >(tee -a /var/log/pp-bootstrap.log) 2>&1
 trap 'touch /var/lib/pp-bootstrap-failed' ERR
+
+# Ubuntu's apt-daily / unattended-upgrades timers fire on boot and hold the apt
+# locks, which makes apt-get die immediately ("Could not get lock") — not just
+# for our bootstrap, but for any user setup command that shells out to apt (e.g.
+# the get.docker.com install script). "mask --now" stops the units if they are
+# already running AND blocks them from being re-triggered for the life of this
+# ephemeral instance. Then wait out anything mid-flight before each apt-get.
+systemctl mask --now apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true
+apt_wait() {
+  for i in $(seq 1 150); do
+    fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 && sleep 2 || break
+  done
+}
+
+apt_wait
 apt-get update -y
-apt-get install -y curl git unzip build-essential
-
-# Docker
-curl -fsSL https://get.docker.com | sh
-systemctl enable docker
-systemctl start docker
-apt-get install -y docker-compose-plugin
-
-# PP deploys as the image's supported SSH user. Give it Docker access and its
-# own Node runtime rather than attempting to override Ubuntu's root SSH policy.
-usermod -aG docker ubuntu
-sudo -u ubuntu -H bash -lc 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash'
-sudo -u ubuntu -H bash -lc 'export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh"; nvm install --lts; nvm alias default lts/*'
+apt_wait
+apt-get install -y ca-certificates curl git unzip
 
 touch /var/lib/pp-bootstrap-done
 `;
@@ -188,6 +201,31 @@ export async function terminateInstance(ec2: EC2Client, instanceId: string): Pro
   await ec2.send(new TerminateInstancesCommand({ InstanceIds: [instanceId] }));
 }
 
+// Waits for an instance to reach the `terminated` state so its ENI is released
+// and the security group no longer has a dependent object. Resolves (rather than
+// throwing) on timeout so callers can still attempt SG deletion with retries.
+export async function waitForInstanceTerminated(
+  ec2: EC2Client,
+  instanceId: string,
+  timeoutMs = 180_000,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    try {
+      const res = await ec2.send(new DescribeInstancesCommand({
+        InstanceIds: [instanceId],
+      }));
+      const state = res.Reservations?.[0]?.Instances?.[0]?.State?.Name;
+      if (!state || state === "terminated") return;
+    } catch (e: any) {
+      // Instance record aged out / never existed — nothing left to wait on.
+      if (e.name === "InvalidInstanceID.NotFound") return;
+      throw e;
+    }
+    await sleep(5000);
+  }
+}
+
 export async function deleteKeyPairAws(ec2: EC2Client, keyName: string): Promise<void> {
   try {
     await ec2.send(new DeleteKeyPairCommand({ KeyName: keyName }));
@@ -202,9 +240,24 @@ export async function deleteSecurityGroupAws(ec2: EC2Client, groupName: string):
       Filters: [{ Name: "group-name", Values: [groupName] }],
     }));
     const groupId = describe.SecurityGroups?.[0]?.GroupId;
-    if (groupId) {
-      await sleep(5000);
-      await ec2.send(new DeleteSecurityGroupCommand({ GroupId: groupId }));
+    if (!groupId) return;
+
+    // The terminated instance's ENI can linger for a while after the instance
+    // itself is gone; DeleteSecurityGroup fails with DependencyViolation until
+    // that ENI is released. Retry with backoff instead of failing outright.
+    const maxAttempts = 12;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        await ec2.send(new DeleteSecurityGroupCommand({ GroupId: groupId }));
+        return;
+      } catch (e: any) {
+        if (e.name === "InvalidGroup.NotFound") return;
+        if (e.name === "DependencyViolation" && attempt < maxAttempts) {
+          await sleep(10_000);
+          continue;
+        }
+        throw e;
+      }
     }
   } catch (e) {
     log.warn({ e, groupName }, "Failed to delete security group");

backend/src/services/gitea.ts

@@ -23,13 +23,61 @@ export async function validateGiteaUrl(url: string): Promise<{ success: boolean;
   }
 }
 
+// Authenticates the PAT and confirms it belongs to the expected account. Gitea 1.19+
+// scopes tokens and does not expose granted scopes via the API, so we cannot pre-check
+// write:issue non-destructively — but authenticating here catches invalid/expired/wrong
+// tokens at setup instead of failing at deploy time.
+export async function validateGiteaToken(
+  url: string,
+  pat: string,
+  expectedUsername: string,
+): Promise<{ success: boolean; error?: string; login?: string }> {
+  try {
+    const res = await axios.get(`${url}/api/v1/user`, {
+      headers: { Authorization: `token ${pat}` },
+      timeout: 10000,
+    });
+    const login: string | undefined = res.data?.login;
+    if (expectedUsername && login && login.toLowerCase() !== expectedUsername.toLowerCase()) {
+      return {
+        success: false,
+        login,
+        error: `Token belongs to "${login}", not "${expectedUsername}". Use a token created by ${expectedUsername}.`,
+      };
+    }
+    return { success: true, login };
+  } catch (e: any) {
+    const status = e?.response?.status;
+    if (status === 401) return { success: false, error: "Token is invalid or expired." };
+    return { success: false, error: e?.message ?? "Token validation failed" };
+  }
+}
+
+// Translates opaque Gitea API errors into actionable messages. A 403 on a write almost
+// always means the PAT lacks the write scope for that resource.
+function giteaWriteError(e: any, action: string): Error {
+  if (e?.response?.status === 403) {
+    return new Error(
+      `Gitea denied ${action} (403 Forbidden). The Personal Access Token is missing write access. ` +
+        `Recreate it with Read and Write permission for the "issue" and "repository" scopes.`,
+    );
+  }
+  return e instanceof Error ? e : new Error(String(e));
+}
+
+// Lists only the repos the authenticated user owns or has access to (owned +
+// collaborator + org member). We deliberately use /user/repos rather than
+// /repos/search: for a Gitea site admin, /repos/search returns EVERY repo on the
+// instance, which would let admins see and manage repos they have no stake in.
+// /user/repos is user-scoped regardless of admin status. Note the shape differs:
+// /user/repos returns a bare array, not the /repos/search { ok, data } envelope.
 export async function fetchUserRepos(user: User): Promise<any[]> {
   const api = giteaApi(user);
   const repos: any[] = [];
   let page = 1;
   while (true) {
-    const res = await api.get(`/repos/search?limit=50&page=${page}`);
-    const data = res.data?.data ?? [];
+    const res = await api.get(`/user/repos?limit=50&page=${page}`);
+    const data = Array.isArray(res.data) ? res.data : [];
     if (data.length === 0) break;
     repos.push(...data);
     if (data.length < 50) break;
@@ -38,16 +86,34 @@ export async function fetchUserRepos(user: User): Promise<any[]> {
   return repos;
 }
 
+export async function fetchRepo(user: User, owner: string, repo: string): Promise<any> {
+  const api = giteaApi(user);
+  const res = await api.get(`/repos/${owner}/${repo}`);
+  return res.data;
+}
+
+// The canonical event set for every PP-managed webhook. Gitea splits PR events:
+// "pull_request" fires on opened/closed/reopened/edited, but a push of new
+// commits to the PR branch fires the SEPARATE "pull_request_sync" event.
+// Without it, the synchronize webhook is never delivered and previews never
+// redeploy on push. "issue_comment" carries the `/pp ...` commands.
+export const PP_WEBHOOK_EVENTS = ["pull_request", "pull_request_sync", "issue_comment"];
+
+// Human-readable label shown in Gitea's webhook list. Applied on creation and
+// backfilled onto older unnamed webhooks during boot reconciliation.
+export const PP_WEBHOOK_NAME = "PR Previews";
+
 export async function registerWebhook(user: User, owner: string, repo: string, webhookUrl: string, secret: string): Promise<number> {
   const api = giteaApi(user);
   const res = await api.post(`/repos/${owner}/${repo}/hooks`, {
     type: "gitea",
+    name: PP_WEBHOOK_NAME,
     config: {
       url: webhookUrl,
       secret,
       content_type: "json",
     },
-    events: ["pull_request", "issue_comment"],
+    events: PP_WEBHOOK_EVENTS,
     active: true,
   });
   return res.data.id;
@@ -60,26 +126,89 @@ export async function deleteWebhook(user: User, owner: string, repo: string, hoo
 
 export async function updateWebhookSecret(user: User, owner: string, repo: string, hookId: string, webhookUrl: string, newSecret: string): Promise<void> {
   const api = giteaApi(user);
+  const hook = await getWebhook(user, owner, repo, hookId);
+  if (!hook) throw new Error("Webhook no longer exists in Gitea");
+
+  const currentEvents: string[] = Array.isArray(hook.events) ? hook.events : [];
   await api.patch(`/repos/${owner}/${repo}/hooks/${hookId}`, {
+    name: PP_WEBHOOK_NAME,
     config: {
       url: webhookUrl,
       secret: newSecret,
       content_type: "json",
     },
-    events: ["pull_request", "issue_comment"],
+    events: [...new Set([...currentEvents, ...PP_WEBHOOK_EVENTS])],
     active: true,
   });
 }
 
+// Fetches a single webhook. Returns null on 404 (webhook deleted in Gitea while
+// PP still has its id on the RepoConfig).
+export async function getWebhook(user: User, owner: string, repo: string, hookId: string): Promise<any | null> {
+  const api = giteaApi(user);
+  try {
+    const res = await api.get(`/repos/${owner}/${repo}/hooks/${hookId}`);
+    return res.data;
+  } catch (e: any) {
+    if (e?.response?.status === 404) return null;
+    throw e;
+  }
+}
+
+export type WebhookReconcileResult =
+  | { status: "missing" }
+  | { status: "ok" }
+  | { status: "patched"; addedEvents: string[]; named: boolean; reactivated: boolean };
+
+// Brings an already-registered webhook up to the current PP standard without
+// tearing it down: unions in any missing events (chiefly "pull_request_sync"
+// for webhooks created before that fix), names it if it was left unnamed, and
+// re-activates it if disabled. User-added extra events are preserved. Patches
+// only when something actually differs, so re-runs are no-ops.
+export async function reconcileWebhook(user: User, owner: string, repo: string, hookId: string): Promise<WebhookReconcileResult> {
+  const hook = await getWebhook(user, owner, repo, hookId);
+  if (!hook) return { status: "missing" };
+
+  const currentEvents: string[] = Array.isArray(hook.events) ? hook.events : [];
+  const missingEvents = PP_WEBHOOK_EVENTS.filter(e => !currentEvents.includes(e));
+  const hasName = typeof hook.name === "string" && hook.name.trim().length > 0;
+  const inactive = hook.active === false;
+
+  if (missingEvents.length === 0 && hasName && !inactive) return { status: "ok" };
+
+  // Gitea's edit-hook API resets an omitted/empty `events` array to push-only,
+  // silently disabling every other trigger. So ANY patch must re-send the full
+  // desired event set (union of existing + PP events) and `active` — never a
+  // partial body that would wipe the triggers we depend on.
+  const patch: any = {
+    events: [...new Set([...currentEvents, ...PP_WEBHOOK_EVENTS])],
+    active: true,
+  };
+  if (!hasName) patch.name = PP_WEBHOOK_NAME;
+
+  const api = giteaApi(user);
+  await api.patch(`/repos/${owner}/${repo}/hooks/${hookId}`, patch);
+
+  return { status: "patched", addedEvents: missingEvents, named: !hasName, reactivated: inactive };
+}
+
 export async function postComment(user: User, owner: string, repo: string, issueNumber: number, body: string): Promise<number> {
   const api = giteaApi(user);
-  const res = await api.post(`/repos/${owner}/${repo}/issues/${issueNumber}/comments`, { body });
-  return res.data.id;
+  try {
+    const res = await api.post(`/repos/${owner}/${repo}/issues/${issueNumber}/comments`, { body });
+    return res.data.id;
+  } catch (e: any) {
+    throw giteaWriteError(e, "posting a PR comment");
+  }
 }
 
 export async function updateComment(user: User, owner: string, repo: string, commentId: number, body: string): Promise<void> {
   const api = giteaApi(user);
-  await api.patch(`/repos/${owner}/${repo}/issues/comments/${commentId}`, { body });
+  try {
+    await api.patch(`/repos/${owner}/${repo}/issues/comments/${commentId}`, { body });
+  } catch (e: any) {
+    throw giteaWriteError(e, "updating a PR comment");
+  }
 }
 
 export async function checkUserPermission(user: User, owner: string, repo: string, username: string): Promise<boolean> {

backend/src/services/ssh.ts

@@ -5,6 +5,11 @@ const log = createLogger("SSH");
 
 export interface SshSession {
   exec(command: string): Promise<{ stdout: string; stderr: string; code: number }>;
+  // Run a long-lived command (e.g. `tail -f`) and receive its stdout/stderr as
+  // it arrives. Unlike exec(), this never buffers or times out — the channel
+  // stays open until the returned handle's close() is called (or the connection
+  // ends). Used to stream live app logs off a running instance.
+  execStream(command: string, onData: (chunk: string) => void): { close(): void };
   close(): void;
   aborted: boolean;
   abort(): void;
@@ -28,6 +33,26 @@ export async function connectSsh(host: string, privateKey: string, maxWaitMs = 3
           if (aborted) throw new Error("SSH session aborted");
           return execOnConn(conn, command);
         },
+        execStream(command: string, onData: (chunk: string) => void) {
+          let stream: any = null;
+          let closed = false;
+          conn.exec(command, (err, s) => {
+            if (err) {
+              if (!closed) onData(`[app-logs] stream error: ${err.message}\n`);
+              return;
+            }
+            if (closed) { try { s.close(); } catch {} return; }
+            stream = s;
+            s.on("data", (d: Buffer) => onData(d.toString()));
+            s.stderr.on("data", (d: Buffer) => onData(d.toString()));
+          });
+          return {
+            close() {
+              closed = true;
+              try { stream?.close(); } catch {}
+            },
+          };
+        },
         close() {
           try { conn.end(); } catch {}
         },
@@ -95,14 +120,37 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st
   });
 }
 
+// The `ubuntu` user accepts SSH almost immediately at boot — long before the
+// cloud-init UserData bootstrap has finished disabling apt-daily and installing
+// the small base dependency set PP needs before SSH-driven setup begins.
+// Proceeding early can still cause apt-lock failures in the deploy steps. Block until the bootstrap writes its
+// done marker; bail out fast (with the tail of its log) if it writes the failure
+// marker instead. Uses short one-shot execs so a stale channel just retries.
 export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise<void> {
   const start = Date.now();
   while (Date.now() - start < maxWaitMs) {
+    if (session.aborted) throw new Error("SSH session aborted");
     try {
-      const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
-      if (res.stdout.trim() === "done") return;
-    } catch {
-      // transient SSH exec error (e.g. timeout, channel reset) — retry
+      const res = await session.exec(
+        "if [ -f /var/lib/pp-bootstrap-failed ]; then echo failed; " +
+        "elif [ -f /var/lib/pp-bootstrap-done ]; then echo done; else echo waiting; fi"
+      );
+      const state = res.stdout.trim();
+      if (state === "done") return;
+      if (state === "failed") {
+        let tail = "";
+        try {
+          const logRes = await session.exec("tail -n 30 /var/log/pp-bootstrap.log 2>/dev/null || true");
+          tail = logRes.stdout.trim();
+        } catch {
+          // best effort — the failure marker alone is enough to abort
+        }
+        throw new Error(`EC2 bootstrap failed${tail ? `:\n${tail}` : ""}`);
+      }
+    } catch (e: any) {
+      // A bootstrap failure we detected above must propagate; only swallow
+      // transient SSH exec errors (timeout, channel reset) and retry.
+      if (e?.message?.startsWith("EC2 bootstrap failed")) throw e;
     }
     await sleep(10_000);
   }

backend/src/services/webhookReconcile.ts

@@ -0,0 +1,48 @@
+import { prisma } from "../lib/db";
+import { createLogger } from "../lib/logger";
+import { reconcileWebhook } from "./gitea";
+
+const log = createLogger("WEBHOOK_RECONCILE");
+
+// On boot, bring every PP-managed Gitea webhook up to the current standard so
+// existing installs self-heal without the operator having to disable/re-enable
+// each repo. Chiefly backfills the "pull_request_sync" event (added after these
+// webhooks were first registered — without it pushes to a PR branch never
+// redeploy) and names any webhook left unnamed. Errors are per-repo so one bad
+// PAT or deleted repo can't abort the whole pass.
+export async function runWebhookReconciliation() {
+  log.info("Reconciling Gitea webhooks");
+
+  const configs = await prisma.repoConfig.findMany({
+    where: { isEnabled: true, giteaWebhookId: { not: null } },
+    include: { user: true },
+  });
+
+  let patched = 0;
+  let named = 0;
+  let missing = 0;
+
+  for (const config of configs) {
+    const { user, repoOwner, repoName, giteaWebhookId } = config;
+    if (!user?.giteaInstanceUrl || !user?.giteaPAT || !giteaWebhookId) continue;
+
+    try {
+      const result = await reconcileWebhook(user as any, repoOwner, repoName, giteaWebhookId);
+      if (result.status === "missing") {
+        missing++;
+        log.warn({ repo: `${repoOwner}/${repoName}`, giteaWebhookId }, "Webhook no longer exists in Gitea");
+      } else if (result.status === "patched") {
+        patched++;
+        if (result.named) named++;
+        log.info(
+          { repo: `${repoOwner}/${repoName}`, addedEvents: result.addedEvents, named: result.named, reactivated: result.reactivated },
+          "Webhook reconciled",
+        );
+      }
+    } catch (e) {
+      log.warn({ e, repo: `${repoOwner}/${repoName}`, userId: user.id }, "Failed to reconcile webhook");
+    }
+  }
+
+  log.info({ total: configs.length, patched, named, missing }, "Webhook reconciliation complete");
+}