Really huge mass update; Getting everything up-to-spec and implementing a wide range of features

2026-07-26 14:24:18 +02:00
parent 2c563685bd
commit 8b53698f29
72 changed files with 4275 additions and 693 deletions
@@ -14,12 +14,14 @@ import { startJobWorker } from "./workers/jobWorker";
 import { startCronWorkers } from "./workers/cronWorker";
 import { getAdminSettings } from "./lib/adminSettings";
 import { runOrphanCleanup } from "./services/orphanCleanup";
+import { runWebhookReconciliation } from "./services/webhookReconcile";

 import { loginHandler, logoutHandler, meHandler, setupStatusHandler, firstUserHandler } from "./routes/auth";
 import { webhookHandler } from "./routes/webhook";
 import { getUserSettings, updateUsername, updatePassword, updateGitea, updateAws, getWebhookSecret, regenerateWebhookSecret } from "./routes/api/user";
 import { listRepos, saveRepoConfig, toggleRepoEnabled, getRepoConfig } from "./routes/api/repos";
-import { listPreviews, getPreview, stopPreviewRoute, previewLogsWs } from "./routes/api/previews";
+import { listPreviews, getPreview, stopPreviewRoute, rebuildPreviewRoute, previewLogsWs, previewAppLogsWs } from "./routes/api/previews";
+import { getStats } from "./routes/api/stats";
 import { listUsers, createUser, updateUser, deleteUser, getSettings, updateSettings, adminListPreviews, adminStopPreview } from "./routes/api/admin";

 const uiBuildPath = join(__dirname, "../../frontend/dist");
@@ -76,16 +78,27 @@ server.path("/", (path) => path
  .http("GET", "/api/repos/{owner}/{repo}/config", (http) => http.onRequest(getRepoConfig))
 );

+// Stats (instance-wide Overview)
+server.path("/", (path) => path
+  .http("GET", "/api/stats", (http) => http.onRequest(getStats))
+);
+
 // Previews
 server.path("/", (path) => path
  .http("GET", "/api/previews", (http) => http.onRequest(listPreviews))
  .http("GET", "/api/previews/{id}", (http) => http.onRequest(getPreview))
  .http("POST", "/api/previews/{id}/stop", (http) => http.onRequest(stopPreviewRoute))
+  .http("POST", "/api/previews/{id}/rebuild", (http) => http.onRequest(rebuildPreviewRoute))
  .ws("/api/previews/{id}/logs", (ws) => ws
    .onOpen(previewLogsWs)
    .onMessage(async () => {})
    .onClose(async () => {})
  )
+  .ws("/api/previews/{id}/applogs", (ws) => ws
+    .onOpen(previewAppLogsWs)
+    .onMessage(async () => {})
+    .onClose(async () => {})
+  )
 );

 // Admin
@@ -127,6 +140,7 @@ server
    startJobWorker();
    startCronWorkers();
    runOrphanCleanup().catch(e => logger.warn(e, "Orphan cleanup error"));
+    runWebhookReconciliation().catch(e => logger.warn(e, "Webhook reconciliation error"));
    logger.info("All workers started");
  })
  .catch((err) => logger.error(err, "Server failed to start"));
@@ -1,9 +1,10 @@
 import { prisma } from "./db";
+import { DEFAULT_INSTANCE_TYPE } from "./instanceTypes";

 export async function getAdminSettings() {
  let settings = await prisma.adminSettings.findUnique({ where: { id: 1 } });
  if (!settings) {
-    settings = await prisma.adminSettings.create({ data: { id: 1 } });
+    settings = await prisma.adminSettings.create({ data: { id: 1, defaultInstanceType: DEFAULT_INSTANCE_TYPE } });
  }
  return settings;
 }
@@ -0,0 +1,71 @@
+import { DEFAULT_INSTANCE_TYPE } from "./instanceTypes";
+
+// Cost estimation for preview EC2 instances.
+//
+// PP bills nothing itself; the EC2 runs in the user's own AWS account, so this
+// is an estimate derived from on-demand pricing and how long each instance ran.
+// It is deliberately not tied to AWS Cost Explorer / billing APIs: those lag by
+// hours-to-a-day and would need extra IAM permissions. Uptime times hourly rate
+// is accurate enough for a dashboard and updates live.
+//
+// Rates below are USD/hour for Linux on-demand in us-east-1. Other regions cost
+// a little more, so treat the number as a lower-bound ballpark. Keep these in
+// sync with the table shown in frontend RepoConfig.tsx.
+
+export const INSTANCE_HOURLY_USD: Record<string, number> = {
+  // Legacy t2 rates are retained for historical previews that were actually
+  // launched as t2 before the default moved to current-generation instances.
+  "t2.micro": 0.0116,
+  "t2.small": 0.023,
+  "t2.medium": 0.0464,
+  "t2.large": 0.0928,
+  "t3.micro": 0.0104,
+  "t3.small": 0.0208,
+  "t3.medium": 0.0416,
+  "t3.large": 0.0832,
+  "t3a.medium": 0.0376,
+  "t3a.large": 0.0752,
+  "t4g.medium": 0.0336,
+  "t4g.large": 0.0672,
+  "m5.large": 0.096,
+  "m5.xlarge": 0.192,
+  "c5.large": 0.085,
+  "c5.xlarge": 0.17,
+};
+
+// Fallback when the instance type is unknown/custom: use the current default so
+// an estimate is still shown rather than $0.
+const DEFAULT_HOURLY_USD = INSTANCE_HOURLY_USD[DEFAULT_INSTANCE_TYPE];
+
+export function hourlyRateUsd(instanceType?: string | null): number {
+  if (!instanceType) return DEFAULT_HOURLY_USD;
+  return INSTANCE_HOURLY_USD[instanceType] ?? DEFAULT_HOURLY_USD;
+}
+
+// Cost of a single instance session from launch to now/termination at the given
+// type's hourly rate. Clamps negatives to 0 in case clocks disagree.
+export function sessionCostUsd(
+  launchedAt: Date,
+  until: Date,
+  instanceType?: string | null,
+): number {
+  const hours = Math.max(0, (until.getTime() - launchedAt.getTime()) / 3_600_000);
+  return hours * hourlyRateUsd(instanceType);
+}
+
+// The fields cost calculation needs off a Preview row.
+export interface PreviewCostFields {
+  accumulatedCostUsd: number;
+  instanceLaunchedAt: Date | null;
+  instanceType: string | null;
+}
+
+// Total estimated cost for a preview: finalized cost of past sessions plus the
+// live session's accrual so far, if an instance is currently running.
+export function computePreviewCostUsd(p: PreviewCostFields, now: Date = new Date()): number {
+  let total = p.accumulatedCostUsd || 0;
+  if (p.instanceLaunchedAt) {
+    total += sessionCostUsd(p.instanceLaunchedAt, now, p.instanceType);
+  }
+  return total;
+}
@@ -0,0 +1,12 @@
+export const DEFAULT_INSTANCE_TYPE = "t3.medium";
+
+export const LEGACY_INSTANCE_TYPE_REPLACEMENTS: Record<string, string> = {
+  "t2.medium": DEFAULT_INSTANCE_TYPE,
+};
+
+export function normalizeInstanceType(value?: unknown): string {
+  const raw = typeof value === "string" ? value.trim() : "";
+  if (!raw) return DEFAULT_INSTANCE_TYPE;
+  if (raw.startsWith("t2.")) return LEGACY_INSTANCE_TYPE_REPLACEMENTS[raw] ?? DEFAULT_INSTANCE_TYPE;
+  return raw.slice(0, 64);
+}
@@ -81,3 +81,38 @@ export const authEnforcementMiddleware = new Middleware<{}, {}>(
  .export();

 export const COOKIE_NAME_EXPORT = COOKIE_NAME;
+
+// The auth-resolution middleware only runs on HTTP requests (`.httpRequest` /
+// `.httpRequestContext`), so `ctr.getAuth()` is never populated on a WebSocket
+// upgrade — every WS would see `success: false` and close 1008. WebSocket
+// handlers must resolve the session themselves; this mirrors the middleware's
+// logic. Reads the session cookie via the ctr cookie API, falling back to
+// parsing the raw `Cookie` header from the upgrade request.
+export async function resolveWsAuth(ctr: any): Promise<AuthState> {
+  let cookieToken: string | undefined;
+  try {
+    cookieToken = ctr.cookies?.get?.(COOKIE_NAME);
+  } catch {
+    // fall through to header parsing
+  }
+  if (!cookieToken) {
+    const raw = (ctr.headers?.get?.("cookie") as string | undefined) || "";
+    const match = raw.match(/(?:^|;\s*)pp_session=([^;]+)/);
+    if (match) cookieToken = decodeURIComponent(match[1]);
+  }
+
+  if (!cookieToken) {
+    return { success: false, message: "No session", tokenProvided: false };
+  }
+
+  const session = await prisma.session.findFirst({
+    where: { hash: cookieToken },
+    include: { user: true },
+  });
+
+  if (!session) {
+    return { success: false, message: "Invalid session", tokenProvided: true };
+  }
+
+  return { success: true, user: session.user, sessionId: session.id };
+}
@@ -0,0 +1,28 @@
+export const PREINSTALL_TOOLS = ["docker", "node", "python", "go", "lua", "build-essential"] as const;
+
+export type PreinstallTool = typeof PREINSTALL_TOOLS[number];
+
+const toolSet = new Set<string>(PREINSTALL_TOOLS);
+const APT_PACKAGE_RE = /^[A-Za-z0-9.+:-]+$/;
+
+export function normalizePreinstallTools(input: unknown, useDockerCompose = false): PreinstallTool[] {
+  const tools = Array.isArray(input)
+    ? input.filter((tool): tool is PreinstallTool => typeof tool === "string" && toolSet.has(tool))
+    : [];
+  if (useDockerCompose && !tools.includes("docker")) tools.push("docker");
+  return [...new Set(tools)];
+}
+
+export function normalizeNodeVersion(input: unknown, nodeSelected: boolean): string | null {
+  if (!nodeSelected) return null;
+  const value = typeof input === "string" ? input.trim() : "";
+  if (!value) return "lts/*";
+  return /^[A-Za-z0-9._/*+-]+$/.test(value) ? value.slice(0, 64) : "lts/*";
+}
+
+export function normalizeAptPackages(input: unknown): string[] {
+  const packages = Array.isArray(input)
+    ? input.filter((pkg): pkg is string => typeof pkg === "string" && APT_PACKAGE_RE.test(pkg))
+    : [];
+  return [...new Set(packages)];
+}
@@ -4,6 +4,8 @@ import { prisma } from "../../lib/db";
 import { makeResponse } from "../../lib/response";
 import { ERROR_MESSAGES } from "../../lib/errors";
 import { getAdminSettings } from "../../lib/adminSettings";
+import { computePreviewCostUsd } from "../../lib/cost";
+import { normalizeInstanceType } from "../../lib/instanceTypes";

 function requireAdmin(ctr: any) {
  const auth = ctr.getAuth?.();
@@ -112,7 +114,7 @@ export async function updateSettings(ctr: any) {
  } = body || {};

  const data: any = {};
-  if (defaultInstanceType) data.defaultInstanceType = defaultInstanceType;
+  if (defaultInstanceType !== undefined) data.defaultInstanceType = normalizeInstanceType(defaultInstanceType);
  if (maxConcurrentInstancesPerUser) data.maxConcurrentInstancesPerUser = Number(maxConcurrentInstancesPerUser);
  if (logSizeLimitBytes) data.logSizeLimitBytes = Number(logSizeLimitBytes);
  if (previewRetentionDays) data.previewRetentionDays = Number(previewRetentionDays);
@@ -147,6 +149,7 @@ export async function adminListPreviews(ctr: any) {
        prNumber: p.prNumber,
        prTitle: p.prTitle,
        status: p.status,
+        stopReason: p.stopReason,
        instanceIp: p.instanceIp,
        port: p.port,
        createdAt: p.createdAt,
@@ -154,6 +157,8 @@ export async function adminListPreviews(ctr: any) {
        repoOwner: p.repoConfig.repoOwner,
        repoName: p.repoConfig.repoName,
        user: (p.repoConfig as any).user,
+        instanceType: p.instanceType,
+        costUsd: computePreviewCostUsd(p),
      }))
    }
  });
@@ -2,6 +2,10 @@ import { prisma } from "../../lib/db";
 import { makeResponse } from "../../lib/response";
 import { ERROR_MESSAGES } from "../../lib/errors";
 import { subscribeToLogs } from "../../services/deploy";
+import { streamAppLogs } from "../../services/appLogs";
+import { resolveWsAuth } from "../../lib/middlewares/auth";
+import { computePreviewCostUsd, hourlyRateUsd } from "../../lib/cost";
+import { fetchRepo } from "../../services/gitea";

 function requireAuth(ctr: any) {
  const auth = ctr.getAuth?.();
@@ -27,6 +31,7 @@ export async function listPreviews(ctr: any) {
        prTitle: p.prTitle,
        commitSha: p.commitSha,
        status: p.status,
+        stopReason: p.stopReason,
        instanceIp: p.instanceIp,
        port: p.port,
        createdAt: p.createdAt,
@@ -34,6 +39,8 @@ export async function listPreviews(ctr: any) {
        lastActivityAt: p.lastActivityAt,
        repoOwner: p.repoConfig.repoOwner,
        repoName: p.repoConfig.repoName,
+        instanceType: p.instanceType,
+        costUsd: computePreviewCostUsd(p),
      }))
    }
  });
@@ -62,6 +69,7 @@ export async function getPreview(ctr: any) {
        prTitle: preview.prTitle,
        commitSha: preview.commitSha,
        status: preview.status,
+        stopReason: preview.stopReason,
        instanceIp: preview.instanceIp,
        port: preview.port,
        logs: preview.logs,
@@ -71,6 +79,9 @@ export async function getPreview(ctr: any) {
        lastActivityAt: preview.lastActivityAt,
        repoOwner: preview.repoConfig.repoOwner,
        repoName: preview.repoConfig.repoName,
+        instanceType: preview.instanceType,
+        costUsd: computePreviewCostUsd(preview),
+        costRateUsd: preview.instanceLaunchedAt ? hourlyRateUsd(preview.instanceType) : 0,
        jobs: preview.jobs.map(j => ({
          id: j.id,
          type: j.type,
@@ -104,8 +115,62 @@ export async function stopPreviewRoute(ctr: any) {
  return makeResponse({ ctr, content: { code: 200, message: "Stop job enqueued" } });
 }

+export async function rebuildPreviewRoute(ctr: any) {
+  const user = requireAuth(ctr);
+  if (!user) return makeResponse({ ctr, content: { code: 401, message: ERROR_MESSAGES.UNAUTHORIZED.message } });
+
+  const id = parseInt(ctr.params.get("id") || "0", 10);
+  const preview = await prisma.preview.findFirst({
+    where: { id, repoConfig: { userId: user.id } },
+    include: { repoConfig: { include: { user: true } } },
+  });
+
+  if (!preview) return makeResponse({ ctr, content: { code: 404, message: ERROR_MESSAGES.NOT_FOUND.message } });
+  if (preview.status === "IGNORED") {
+    return makeResponse({ ctr, content: { code: 400, message: "Ignored previews cannot be rebuilt" } });
+  }
+
+  const fullUser = preview.repoConfig.user;
+  if (!fullUser.giteaInstanceUrl || !fullUser.giteaPAT) {
+    return makeResponse({ ctr, content: { code: 400, message: "Gitea credentials not configured" } });
+  }
+
+  const repo = await fetchRepo(fullUser as any, preview.repoConfig.repoOwner, preview.repoConfig.repoName);
+  const cloneUrl = repo.clone_url;
+  if (!cloneUrl) {
+    return makeResponse({ ctr, content: { code: 400, message: "Gitea repo clone URL not found" } });
+  }
+
+  const isFirstDeploy = preview.status === "STOPPED";
+  if (isFirstDeploy) {
+    await prisma.preview.update({
+      where: { id: preview.id },
+      data: { status: "PROVISIONING", stopReason: null, stoppedAt: null, lastActivityAt: new Date() },
+    });
+  } else {
+    await prisma.preview.update({ where: { id: preview.id }, data: { lastActivityAt: new Date() } });
+  }
+
+  await prisma.job.create({
+    data: {
+      previewId: preview.id,
+      type: "DEPLOY",
+      status: "PENDING",
+      payload: {
+        commitSha: preview.commitSha,
+        prNumber: preview.prNumber,
+        prTitle: preview.prTitle,
+        cloneUrl,
+        isFirstDeploy,
+      },
+    },
+  });
+
+  return makeResponse({ ctr, content: { code: 200, message: "Rebuild job enqueued" } });
+}
+
 export async function previewLogsWs(ctr: any) {
-  const auth = ctr.getAuth?.();
+  const auth = await resolveWsAuth(ctr);
  if (!auth?.success) {
    ctr.close(1008, "Unauthorized");
    return;
@@ -121,13 +186,45 @@ export async function previewLogsWs(ctr: any) {
    return;
  }

-  await ctr.print(JSON.stringify({ type: "init", logs: preview.logs }));
+  // rjweb's WS print signature is print(type, content) — content objects are
+  // JSON-serialized for us. Passing a single stringified arg puts the JSON in
+  // the `type` slot and sends an empty frame, so always use ("text", obj).
+  await ctr.print("text", { type: "init", logs: preview.logs });

  const unsub = subscribeToLogs(id, (text) => {
    try {
-      ctr.print(JSON.stringify({ type: "append", text }));
+      ctr.print("text", { type: "append", text });
    } catch {}
  });

  ctr.$abort(unsub);
 }
+
+// Live stream of the running app's OWN stdout/stderr (not the PP deploy log).
+// Opens a dedicated SSH tail against the instance for the duration of the
+// connection; nothing is persisted. Only available while the preview is live.
+export async function previewAppLogsWs(ctr: any) {
+  const auth = await resolveWsAuth(ctr);
+  if (!auth?.success) {
+    ctr.close(1008, "Unauthorized");
+    return;
+  }
+
+  const id = parseInt(ctr.params.get("id") || "0", 10);
+  const preview = await prisma.preview.findFirst({
+    where: { id, repoConfig: { userId: auth.user.id } },
+  });
+
+  if (!preview) {
+    ctr.close(1008, "Not found");
+    return;
+  }
+
+  const stop = await streamAppLogs(id, (text) => {
+    try {
+      ctr.print("text", { type: "append", text });
+    } catch {}
+  });
+
+  ctr.$abort(stop);
+}
@@ -4,6 +4,9 @@ import { ERROR_MESSAGES } from "../../lib/errors";
 import { fetchUserRepos, registerWebhook, deleteWebhook } from "../../services/gitea";
 import { getAdminSettings } from "../../lib/adminSettings";
 import { env } from "../../lib/env";
+import { computePreviewCostUsd } from "../../lib/cost";
+import { normalizeInstanceType } from "../../lib/instanceTypes";
+import { normalizeAptPackages, normalizeNodeVersion, normalizePreinstallTools } from "../../lib/preinstall";

 function requireAuth(ctr: any) {
  const auth = ctr.getAuth?.();
@@ -24,6 +27,16 @@ export async function listRepos(ctr: any) {
  const configs = await prisma.repoConfig.findMany({ where: { userId: user.id } });
  const configMap = new Map(configs.map(c => [`${c.repoOwner}/${c.repoName}`, c]));

+  // Estimated total EC2 cost per repo — sum over that repo's previews. See lib/cost.ts.
+  const costPreviews = await prisma.preview.findMany({
+    where: { repoConfig: { userId: user.id } },
+    select: { repoConfigId: true, accumulatedCostUsd: true, instanceLaunchedAt: true, instanceType: true },
+  });
+  const costByConfigId = new Map<number, number>();
+  for (const p of costPreviews) {
+    costByConfigId.set(p.repoConfigId, (costByConfigId.get(p.repoConfigId) ?? 0) + computePreviewCostUsd(p));
+  }
+
  const allOwners = [...new Set(giteaRepos.map((r: any) => r.full_name?.split("/")[0]).filter(Boolean))];
  const allConfigs = await prisma.repoConfig.findMany({
    where: { repoOwner: { in: allOwners } },
@@ -43,10 +56,20 @@ export async function listRepos(ctr: any) {
      name,
      fullName: r.full_name,
      htmlUrl: r.html_url,
+      isPrivate: Boolean(r.private),
      isEnabled: config?.isEnabled ?? false,
      claimedByOther: claimedByOthers.has(key),
      config: config ? safeConfig : null,
+      costUsd: config ? (costByConfigId.get(config.id) ?? 0) : 0,
    };
+  }).sort((a: any, b: any) => {
+    const configuredDelta = Number(Boolean(b.config)) - Number(Boolean(a.config));
+    if (configuredDelta !== 0) return configuredDelta;
+
+    const ownerDelta = a.owner.localeCompare(b.owner, undefined, { sensitivity: "base", numeric: true });
+    if (ownerDelta !== 0) return ownerDelta;
+
+    return a.name.localeCompare(b.name, undefined, { sensitivity: "base", numeric: true });
  });

  return makeResponse({ ctr, content: { code: 200, data: result } });
@@ -86,22 +109,29 @@ export async function saveRepoConfig(ctr: any) {

  const settings = await getAdminSettings();

+  const useDockerCompose = Boolean(configData.useDockerCompose ?? false);
+  const preinstallTools = normalizePreinstallTools(configData.preinstallTools, useDockerCompose);
  const sanitized = {
    repoOwner: owner,
    repoName: repo,
    userId: user.id,
-    instanceType: configData.instanceType ?? settings.defaultInstanceType,
+    instanceType: normalizeInstanceType(configData.instanceType ?? settings.defaultInstanceType),
    inactivityHours: Math.min(72, Math.max(0.5, Number(configData.inactivityHours ?? 12))),
    port: Number(configData.port ?? 3000),
    envVars: configData.envVars ?? {},
-    useDockerCompose: Boolean(configData.useDockerCompose ?? false),
+    preinstallTools,
+    nodeVersion: normalizeNodeVersion(configData.nodeVersion, preinstallTools.includes("node")),
+    useDockerCompose,
    composeFilePath: configData.composeFilePath ?? null,
-    aptPackages: Array.isArray(configData.aptPackages) ? configData.aptPackages : [],
+    aptPackages: normalizeAptPackages(configData.aptPackages),
    setupCommands: Array.isArray(configData.setupCommands) ? configData.setupCommands : [],
    buildCommands: Array.isArray(configData.buildCommands) ? configData.buildCommands : [],
    postBuildCommands: Array.isArray(configData.postBuildCommands) ? configData.postBuildCommands : [],
    runCommand: configData.runCommand ?? null,
    denyList: Array.isArray(configData.denyList) ? configData.denyList : [],
+    disabledCommands: Array.isArray(configData.disabledCommands)
+      ? configData.disabledCommands.filter((c: unknown): c is string => typeof c === "string" && c !== "help")
+      : [],
  };

  let config;
@@ -0,0 +1,91 @@
+import { prisma } from "../../lib/db";
+import { makeResponse } from "../../lib/response";
+import { ERROR_MESSAGES } from "../../lib/errors";
+import { computePreviewCostUsd, hourlyRateUsd } from "../../lib/cost";
+import { PreviewStatus } from "@prisma/client";
+
+function requireAuth(ctr: any) {
+  const auth = ctr.getAuth?.();
+  if (!auth?.success) return null;
+  return auth.user;
+}
+
+const ACTIVE_STATUSES: PreviewStatus[] = ["PROVISIONING", "BUILDING", "RUNNING"];
+
+// Instance-wide ("global") stats for the Overview homepage.
+export async function getStats(ctr: any) {
+  const user = requireAuth(ctr);
+  if (!user) return makeResponse({ ctr, content: { code: 401, message: ERROR_MESSAGES.UNAUTHORIZED.message } });
+
+  const [
+    totalUsers,
+    totalRepos,
+    enabledRepos,
+    totalPreviews,
+    activePreviews,
+    activeInstances,
+    byStatusRaw,
+    recentRaw,
+    costRaw,
+  ] = await Promise.all([
+    prisma.user.count(),
+    prisma.repoConfig.count(),
+    prisma.repoConfig.count({ where: { isEnabled: true } }),
+    prisma.preview.count(),
+    prisma.preview.count({ where: { status: { in: ACTIVE_STATUSES } } }),
+    // A live EC2 instance is any preview still holding an instanceId.
+    prisma.preview.count({ where: { instanceId: { not: null } } }),
+    prisma.preview.groupBy({ by: ["status"], _count: { _all: true } }),
+    prisma.preview.findMany({
+      orderBy: { updatedAt: "desc" },
+      take: 8,
+      include: { repoConfig: { select: { repoOwner: true, repoName: true } } },
+    }),
+    prisma.preview.findMany({
+      select: { accumulatedCostUsd: true, instanceLaunchedAt: true, instanceType: true },
+    }),
+  ]);
+
+  const byStatus: Record<string, number> = {};
+  for (const row of byStatusRaw) byStatus[row.status] = row._count._all;
+
+  // Instance-wide (this PP deployment) estimated EC2 spend, and the current
+  // hourly burn rate across every live instance. See lib/cost.ts.
+  const now = new Date();
+  let totalCostUsd = 0;
+  let hourlyBurnUsd = 0;
+  for (const p of costRaw) {
+    totalCostUsd += computePreviewCostUsd(p, now);
+    if (p.instanceLaunchedAt) hourlyBurnUsd += hourlyRateUsd(p.instanceType);
+  }
+
+  return makeResponse({
+    ctr,
+    content: {
+      code: 200,
+      data: {
+        totalUsers,
+        totalRepos,
+        enabledRepos,
+        totalPreviews,
+        activePreviews,
+        activeInstances,
+        totalCostUsd,
+        hourlyBurnUsd,
+        byStatus,
+        recentPreviews: recentRaw.map(p => ({
+          id: p.id,
+          prNumber: p.prNumber,
+          prTitle: p.prTitle,
+          status: p.status,
+          stopReason: p.stopReason,
+          instanceIp: p.instanceIp,
+          port: p.port,
+          updatedAt: p.updatedAt,
+          repoOwner: p.repoConfig.repoOwner,
+          repoName: p.repoConfig.repoName,
+        })),
+      },
+    },
+  });
+}
@@ -4,7 +4,7 @@ import { prisma } from "../../lib/db";
 import { makeResponse } from "../../lib/response";
 import { ERROR_MESSAGES } from "../../lib/errors";
 import { encrypt, decrypt } from "../../lib/encryption";
-import { validateGiteaUrl } from "../../services/gitea";
+import { updateWebhookSecret, validateGiteaUrl, validateGiteaToken } from "../../services/gitea";
 import { validateAwsCredentials } from "../../services/ec2";
 import { env } from "../../lib/env";

@@ -14,6 +14,39 @@ function requireAuth(ctr: any) {
  return auth.user;
 }

+async function getOrCreateWebhookToken(userId: number) {
+  const existing = await prisma.webhookToken.findUnique({ where: { userId } });
+  if (existing) return existing;
+
+  const secret = randomBytes(32).toString("hex");
+  return prisma.webhookToken.create({ data: { userId, token: secret } });
+}
+
+async function syncRegisteredRepoWebhooks(userId: number, fullUser: any, secret: string) {
+  if (!fullUser?.giteaInstanceUrl || !fullUser?.giteaPAT) {
+    return { updated: 0, errors: [] as string[] };
+  }
+
+  const configs = await prisma.repoConfig.findMany({
+    where: { userId, giteaWebhookId: { not: null } },
+  });
+
+  const webhookUrl = `${env.PP_BASE_URL}/webhook/${userId}`;
+  const errors: string[] = [];
+  let updated = 0;
+
+  for (const config of configs) {
+    try {
+      await updateWebhookSecret(fullUser, config.repoOwner, config.repoName, config.giteaWebhookId!, webhookUrl, secret);
+      updated++;
+    } catch (e: any) {
+      errors.push(`${config.repoOwner}/${config.repoName}: ${e.message}`);
+    }
+  }
+
+  return { updated, errors };
+}
+
 export async function getUserSettings(ctr: any) {
  const user = requireAuth(ctr);
  if (!user) return makeResponse({ ctr, content: { code: ERROR_MESSAGES.UNAUTHORIZED.code, message: ERROR_MESSAGES.UNAUTHORIZED.message } });
@@ -88,17 +121,38 @@ export async function updateGitea(ctr: any) {
  }

  const data: any = { giteaInstanceUrl: cleanUrl, giteaUsername };
-  if (giteaPAT) data.giteaPAT = encrypt(giteaPAT);
+  if (giteaPAT) {
+    const tokenCheck = await validateGiteaToken(cleanUrl, giteaPAT, giteaUsername);
+    if (!tokenCheck.success) {
+      return makeResponse({ ctr, content: { code: 400, message: `Gitea token rejected: ${tokenCheck.error}` } });
+    }
+    data.giteaPAT = encrypt(giteaPAT);
+  }

-  await prisma.user.update({ where: { id: user.id }, data });
-  return makeResponse({ ctr, content: { code: 200, message: `Connected to Gitea ${validation.version}`, data: { version: validation.version } } });
+  const updatedUser = await prisma.user.update({ where: { id: user.id }, data });
+  const webhookToken = await getOrCreateWebhookToken(user.id);
+  const hookSync = await syncRegisteredRepoWebhooks(user.id, updatedUser, webhookToken.token);
+
+  return makeResponse({
+    ctr, content: {
+      code: 200,
+      message: hookSync.errors.length
+        ? `Connected to Gitea ${validation.version}; ${hookSync.updated} webhooks updated, ${hookSync.errors.length} failed`
+        : hookSync.updated > 0
+          ? `Connected to Gitea ${validation.version}; ${hookSync.updated} webhooks updated`
+          : `Connected to Gitea ${validation.version}`,
+      data: { version: validation.version, hookSync },
+    }
+  });
 }

 export async function updateAws(ctr: any) {
  const user = requireAuth(ctr);
  if (!user) return makeResponse({ ctr, content: { code: 401, message: ERROR_MESSAGES.UNAUTHORIZED.message } });
  const body = await ctr.body();
-  const { awsAccessKeyId, awsSecretAccessKey, awsRegion } = body || {};
+  const awsAccessKeyId = typeof body?.awsAccessKeyId === "string" ? body.awsAccessKeyId.trim() : body?.awsAccessKeyId;
+  const awsSecretAccessKey = typeof body?.awsSecretAccessKey === "string" ? body.awsSecretAccessKey.trim() : body?.awsSecretAccessKey;
+  const awsRegion = typeof body?.awsRegion === "string" ? body.awsRegion.trim() : body?.awsRegion;

  if (!awsAccessKeyId || !awsSecretAccessKey || !awsRegion) {
    return makeResponse({ ctr, content: { code: 400, message: "AWS credentials and region required" } });
@@ -131,13 +185,9 @@ export async function getWebhookSecret(ctr: any) {
  const user = requireAuth(ctr);
  if (!user) return makeResponse({ ctr, content: { code: 401, message: ERROR_MESSAGES.UNAUTHORIZED.message } });

-  let token = await prisma.webhookToken.findUnique({ where: { userId: user.id } });
-  if (!token) {
-    const secret = randomBytes(32).toString("hex");
-    token = await prisma.webhookToken.create({ data: { userId: user.id, token: secret } });
-  }
+  const token = await getOrCreateWebhookToken(user.id);

-  return makeResponse({ ctr, content: { code: 200, data: { token: token.token } } });
+  return makeResponse({ ctr, content: { code: 200, data: { token: token.token, webhookUrl: `${env.PP_BASE_URL}/webhook/${user.id}` } } });
 }

 export async function regenerateWebhookSecret(ctr: any) {
@@ -156,27 +206,13 @@ export async function regenerateWebhookSecret(ctr: any) {
    return makeResponse({ ctr, content: { code: 200, message: "Secret regenerated (no Gitea hooks to update)", data: { token: newSecret } } });
  }

-  const configs = await prisma.repoConfig.findMany({
-    where: { userId: user.id, giteaWebhookId: { not: null } },
-  });
-
-  const { updateWebhookSecret } = await import("../../services/gitea");
-  const webhookUrl = `${env.PP_BASE_URL}/webhook/${user.id}`;
-  const errors: string[] = [];
-
-  for (const config of configs) {
-    try {
-      await updateWebhookSecret(fullUser as any, config.repoOwner, config.repoName, config.giteaWebhookId!, webhookUrl, newSecret);
-    } catch (e: any) {
-      errors.push(`${config.repoOwner}/${config.repoName}: ${e.message}`);
-    }
-  }
+  const hookSync = await syncRegisteredRepoWebhooks(user.id, fullUser, newSecret);

  return makeResponse({
    ctr, content: {
      code: 200,
-      message: errors.length ? `Secret regenerated with ${errors.length} hook update errors` : "Secret regenerated and all hooks updated",
-      data: { token: newSecret, errors },
+      message: hookSync.errors.length ? `Secret regenerated with ${hookSync.errors.length} hook update errors` : "Secret regenerated and all hooks updated",
+      data: { token: newSecret, errors: hookSync.errors, hookSync },
    }
  });
 }
@@ -133,7 +133,9 @@ async function handlePullRequestEvent(user: any, payload: any) {
        },
      });
    }
-  } else if (action === "synchronize") {
+  } else if (action === "synchronized" || action === "synchronize") {
+    // Gitea sends "synchronized" (past tense) on a push to the PR branch;
+    // GitHub uses "synchronize". Accept both so the redeploy fires either way.
    const preview = await prisma.preview.findFirst({
      where: { repoConfigId: repoConfig.id, prNumber },
      orderBy: { createdAt: "desc" },
@@ -186,7 +188,12 @@ async function handleIssueCommentEvent(user: any, payload: any) {
  const repoName = repo.name;
  const prNumber = issue.number;

-  if (user.giteaUsername && comment.user?.login === user.giteaUsername) return;
+  // Note: we intentionally do NOT ignore comments authored by PP's own Gitea
+  // account here. Loop-prevention is already handled by the `/pp ` prefix check
+  // above — PP's own comments (status updates, `/pp logs` output) never start
+  // with `/pp `, so they can't re-trigger a command. Guarding on giteaUsername
+  // instead only blocked legitimate commands from operators who run PP under
+  // the same account they comment from.

  const repoConfig = await prisma.repoConfig.findFirst({
    where: { repoOwner: owner, repoName, userId: user.id, isEnabled: true },
@@ -207,8 +214,9 @@ async function handleIssueCommentEvent(user: any, payload: any) {
    data: { lastActivityAt: new Date() },
  });

+  // First token after "/pp " is the command word; everything else is ignored.
  const commandLine = body.trimStart().split("\n")[0].trim();
-  const command = commandLine.replace("/pp ", "").trim();
+  const command = commandLine.replace(/^\/pp\s+/, "").trim().split(/\s+/)[0].toLowerCase();

  const preview = await prisma.preview.findFirst({
    where: { repoConfigId: repoConfig.id, prNumber },
@@ -217,21 +225,45 @@ async function handleIssueCommentEvent(user: any, payload: any) {

  const cloneUrl = repo.clone_url;

-  if (command === "rebuild") {
-    if (!preview || (preview.status !== "RUNNING" && preview.status !== "FAILED")) return;
+  // Operator-configured per-repo command switches. `help` can never be disabled
+  // so users always retain a way to discover what is (and isn't) available.
+  const disabled: string[] = Array.isArray(repoConfig.disabledCommands) ? repoConfig.disabledCommands : [];
+  if (command !== "help" && disabled.includes(command)) {
+    await ack(user, owner, repoName, prNumber, `⚠️ The \`/pp ${command}\` command is disabled for this repository.`);
+    return;
+  }
+
+  if (command === "help") {
+    await ack(user, owner, repoName, prNumber, buildHelpBody(disabled));
+  } else if (command === "rebuild") {
+    if (!preview) {
+      await ack(user, owner, repoName, prNumber, "ℹ️ No preview exists for this PR yet. Use `/pp start` to create one.");
+      return;
+    }
+    if (preview.status === "IGNORED") return;
+    if (preview.status === "STOPPED") {
+      // Nothing live to reuse — re-provision a fresh instance (first deploy).
+      await prisma.preview.update({ where: { id: preview.id }, data: { status: "PROVISIONING", stopReason: null, stoppedAt: null } });
+      await prisma.job.create({
+        data: { previewId: preview.id, type: "DEPLOY", status: "PENDING", payload: { commitSha: preview.commitSha, prNumber, prTitle: preview.prTitle, cloneUrl, isFirstDeploy: true } },
+      });
+      await ack(user, owner, repoName, prNumber, "🔄 Preview was stopped — re-provisioning a fresh instance...");
+      return;
+    }
+    // RUNNING / FAILED / BUILDING / PROVISIONING: rebuild on the existing instance.
    await prisma.job.create({
-      data: {
-        previewId: preview.id,
-        type: "DEPLOY",
-        status: "PENDING",
-        payload: { commitSha: preview.commitSha, prNumber, prTitle: preview.prTitle, cloneUrl, isFirstDeploy: false },
-      },
+      data: { previewId: preview.id, type: "DEPLOY", status: "PENDING", payload: { commitSha: preview.commitSha, prNumber, prTitle: preview.prTitle, cloneUrl, isFirstDeploy: false } },
    });
+    await ack(user, owner, repoName, prNumber, "🔄 Rebuilding preview on the existing instance...");
  } else if (command === "stop") {
-    if (!preview || preview.status === "STOPPED") return;
+    if (!preview || preview.status === "STOPPED") {
+      await ack(user, owner, repoName, prNumber, "ℹ️ No running preview to stop.");
+      return;
+    }
    await prisma.job.create({
      data: { previewId: preview.id, type: "STOP", status: "PENDING", payload: { reason: "Manual stop" } },
    });
+    await ack(user, owner, repoName, prNumber, "🛑 Stopping preview...");
  } else if (command === "start") {
    if (!preview) {
      const newPreview = await prisma.preview.create({
@@ -240,14 +272,21 @@ async function handleIssueCommentEvent(user: any, payload: any) {
      await prisma.job.create({
        data: { previewId: newPreview.id, type: "DEPLOY", status: "PENDING", payload: { commitSha: "", prNumber, prTitle: issue.title, cloneUrl, isFirstDeploy: true } },
      });
+      await ack(user, owner, repoName, prNumber, "🚀 Starting preview...");
    } else if (preview.status === "STOPPED" || preview.status === "IGNORED") {
-      await prisma.preview.update({ where: { id: preview.id }, data: { status: "PROVISIONING" } });
+      await prisma.preview.update({ where: { id: preview.id }, data: { status: "PROVISIONING", stopReason: null, stoppedAt: null } });
      await prisma.job.create({
        data: { previewId: preview.id, type: "DEPLOY", status: "PENDING", payload: { commitSha: preview.commitSha, prNumber, prTitle: preview.prTitle, cloneUrl, isFirstDeploy: true } },
      });
+      await ack(user, owner, repoName, prNumber, "🚀 Starting preview...");
+    } else {
+      await ack(user, owner, repoName, prNumber, `ℹ️ Preview is already \`${preview.status.toLowerCase()}\`.`);
    }
  } else if (command === "logs") {
-    if (!preview) return;
+    if (!preview) {
+      await ack(user, owner, repoName, prNumber, "ℹ️ No preview exists for this PR yet.");
+      return;
+    }
    const lastLines = (preview.logs || "").split("\n").slice(-50).join("\n");
    const logBody = `**PP Logs** (last 50 lines)\n\`\`\`\n${lastLines}\n\`\`\``;
    await postComment(user, owner, repoName, prNumber, logBody);
@@ -259,9 +298,48 @@ async function handleIssueCommentEvent(user: any, payload: any) {
    } else {
      await prisma.preview.update({ where: { id: preview.id }, data: { status: "IGNORED" } });
    }
+    await ack(user, owner, repoName, prNumber, "🔕 This PR is now ignored. Future pushes and commands (except `/pp start`) will be skipped.");
+  } else {
+    await ack(user, owner, repoName, prNumber, `❓ Unknown command \`/pp ${command}\`. Try \`/pp help\`.`);
  }
 }

+// Posts a short acknowledgement comment for a command. Best-effort: a Gitea
+// failure here must never bubble up and break command processing.
+async function ack(user: any, owner: string, repo: string, prNumber: number, message: string): Promise<void> {
+  try {
+    await postComment(user, owner, repo, prNumber, message);
+  } catch (e) {
+    log.warn({ e, owner, repo, prNumber }, "Failed to post command acknowledgement comment");
+  }
+}
+
+// The canonical `/pp` command list, also used to render `/pp help`.
+export const PP_COMMANDS: { name: string; usage: string; description: string }[] = [
+  { name: "rebuild", usage: "/pp rebuild", description: "Rebuild the preview — reuses the running instance, or re-provisions if stopped." },
+  { name: "stop", usage: "/pp stop", description: "Stop and terminate the preview instance." },
+  { name: "start", usage: "/pp start", description: "Start a stopped/ignored preview, or create one if none exists." },
+  { name: "logs", usage: "/pp logs", description: "Post the last 50 lines of build/run logs as a comment." },
+  { name: "ignore", usage: "/pp ignore", description: "Ignore this PR — skip future pushes and commands (except /pp start)." },
+  { name: "help", usage: "/pp help", description: "Show this command reference." },
+];
+
+function buildHelpBody(disabled: string[]): string {
+  const rows = PP_COMMANDS.map(c => {
+    const off = c.name !== "help" && disabled.includes(c.name);
+    const cmd = off ? `~~\`${c.usage}\`~~` : `\`${c.usage}\``;
+    const desc = off ? `_(disabled for this repo)_ ${c.description}` : c.description;
+    return `| ${cmd} | ${desc} |`;
+  }).join("\n");
+  return `## 🤖 PR Previews — Commands
+
+Only the PR author or a repo owner/admin can run these.
+
+| Command | Description |
+| --- | --- |
+${rows}`;
+}
+
 async function isRepoAdmin(user: any, owner: string, repo: string, username: string): Promise<boolean> {
  try {
    const { getRepoCollaboratorPermission } = await import("../services/gitea");
@@ -0,0 +1,133 @@
+import { prisma } from "../lib/db";
+import { decrypt } from "../lib/encryption";
+import { connectSsh, type SshSession } from "./ssh";
+import { createLogger } from "../lib/logger";
+
+const log = createLogger("APPLOGS");
+
+// The live app-log stream is deliberately separate from a preview's persisted
+// `logs` (which hold the PP deploy engine's own output). The running app's
+// stdout/stderr lives on the EC2 instance — in /opt/app/pp.log for a plain
+// process run, or in the container logs for docker-compose mode — so we tail it
+// on demand over an SSH channel while clients are watching. Nothing here is
+// written to the database; it's a passthrough to the WebSocket.
+//
+// A single tail per preview is shared (multiplexed) across all connected
+// viewers: the first subscriber opens the SSH connection, later ones attach to
+// it, and the connection is torn down once the last viewer disconnects. Recent
+// output is kept in a ring buffer so a late joiner sees history immediately
+// instead of a blank pane until the next line arrives.
+
+function tailCommand(useDockerCompose: boolean, composeFilePath: string | null): string {
+  if (useDockerCompose) {
+    const composePath = composeFilePath || "docker-compose.yml";
+    // `--tail=200` seeds recent history so the viewer isn't blank on connect;
+    // `-f` follows new output. Compose writes container logs to stdout/stderr,
+    // both of which execStream captures.
+    return `cd /opt/app && sudo docker compose -f '${composePath.replace(/'/g, `'"'"'`)}' logs -f --tail=200`;
+  }
+  // `-F` (not `-f`) follows by name and retries if the file is missing or gets
+  // rotated — the app may not have written pp.log yet when a client connects.
+  return `tail -n 200 -F /opt/app/pp.log 2>&1`;
+}
+
+const RING_BUFFER_BYTES = 64 * 1024;
+
+interface StreamEntry {
+  subscribers: Set<(chunk: string) => void>;
+  buffer: string[]; // recent chunks, replayed to late joiners
+  bufferBytes: number;
+  connecting: Promise<void>;
+  session: SshSession | null;
+  streamHandle: { close(): void } | null;
+  torndown: boolean;
+}
+
+const streams = new Map<number, StreamEntry>();
+
+function pushToBuffer(entry: StreamEntry, chunk: string) {
+  entry.buffer.push(chunk);
+  entry.bufferBytes += Buffer.byteLength(chunk, "utf8");
+  while (entry.bufferBytes > RING_BUFFER_BYTES && entry.buffer.length > 1) {
+    const dropped = entry.buffer.shift()!;
+    entry.bufferBytes -= Buffer.byteLength(dropped, "utf8");
+  }
+}
+
+function broadcast(entry: StreamEntry, chunk: string) {
+  pushToBuffer(entry, chunk);
+  for (const cb of entry.subscribers) {
+    try { cb(chunk); } catch {}
+  }
+}
+
+async function openStream(previewId: number, entry: StreamEntry) {
+  const preview = await prisma.preview.findUnique({
+    where: { id: previewId },
+    include: { repoConfig: true },
+  });
+
+  if (!preview || !preview.instanceIp || !preview.sshPrivateKey) {
+    broadcast(entry, "[app-logs] No running instance for this preview — app logs are only available while it is live.\n");
+    return;
+  }
+
+  const privateKey = decrypt(preview.sshPrivateKey);
+  const cmd = tailCommand(preview.repoConfig.useDockerCompose, preview.repoConfig.composeFilePath);
+
+  try {
+    const session = await connectSsh(preview.instanceIp, privateKey, 30_000);
+    // The last viewer may have left while we were connecting.
+    if (entry.torndown || entry.subscribers.size === 0) {
+      session.close();
+      return;
+    }
+    entry.session = session;
+    entry.streamHandle = session.execStream(cmd, (chunk) => broadcast(entry, chunk));
+  } catch (e: any) {
+    broadcast(entry, `[app-logs] Failed to connect to instance: ${e.message}\n`);
+    log.warn({ previewId, error: e.message }, "App log stream failed to connect");
+  }
+}
+
+function teardown(previewId: number, entry: StreamEntry) {
+  entry.torndown = true;
+  streams.delete(previewId);
+  try { entry.streamHandle?.close(); } catch {}
+  try { entry.session?.close(); } catch {}
+}
+
+// Subscribe to a preview's live app logs. The returned function unsubscribes;
+// when the last subscriber leaves, the shared SSH tail is torn down.
+export async function streamAppLogs(
+  previewId: number,
+  onData: (chunk: string) => void,
+): Promise<() => void> {
+  let entry = streams.get(previewId);
+
+  if (!entry) {
+    entry = {
+      subscribers: new Set(),
+      buffer: [],
+      bufferBytes: 0,
+      connecting: Promise.resolve(),
+      session: null,
+      streamHandle: null,
+      torndown: false,
+    };
+    streams.set(previewId, entry);
+    entry.connecting = openStream(previewId, entry);
+  }
+
+  entry.subscribers.add(onData);
+  // Replay buffered history so a late joiner isn't staring at a blank pane.
+  for (const chunk of entry.buffer) {
+    try { onData(chunk); } catch {}
+  }
+
+  const current = entry;
+  return () => {
+    current.subscribers.delete(onData);
+    if (current.subscribers.size === 0) teardown(previewId, current);
+  };
+}
@@ -10,10 +10,13 @@ import {
  launchInstance,
  waitForInstanceRunning,
  terminateInstance,
+  waitForInstanceTerminated,
  deleteKeyPairAws,
  deleteSecurityGroupAws,
 } from "./ec2";
-import { connectSsh, type SshSession } from "./ssh";
+import { connectSsh, waitForBootstrap, type SshSession } from "./ssh";
+import { computePreviewCostUsd, sessionCostUsd } from "../lib/cost";
+import { normalizeAptPackages, normalizePreinstallTools } from "../lib/preinstall";
 import {
  buildPrCommentBody,
  postComment,
@@ -23,6 +26,15 @@ import type { Preview, RepoConfig, User } from "@prisma/client";

 const log = createLogger("DEPLOY");

+// Block until Ubuntu's apt-daily / unattended-upgrades timers release every apt
+// lock, so a following `apt-get update`/`install` doesn't die with "Could not
+// get lock". Polls all four lock files for up to 300s (150 * 2s). `fuser`
+// returns 0 while a file is in use, non-zero once it's free.
+const APT_WAIT =
+  `sudo bash -c 'for i in $(seq 1 150); do ` +
+  `fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 ` +
+  `&& sleep 2 || break; done'`;
+
 const activeSshSessions = new Map<number, SshSession>();

 export function abortJobForPreview(previewId: number) {
@@ -33,12 +45,23 @@ export function abortJobForPreview(previewId: number) {
  }
 }

+// Prefix every non-empty line of a log chunk with a local-time `[HH:MM:SS]`
+// stamp. SSH execs buffer their whole output and hand it to appendLog at once,
+// so a command's lines share the stamp of the moment it completed — the useful
+// signal is *when each step happened*, which this captures. Blank lines (e.g.
+// the `\n---\n` separators) are left untouched so the log keeps its spacing.
+function stampLines(text: string): string {
+  const stamp = `[${new Date().toTimeString().slice(0, 8)}] `;
+  return text.replace(/^(?=.)/gm, stamp);
+}
+
 export async function appendLog(previewId: number, text: string) {
  const settings = await getAdminSettings();
  const preview = await prisma.preview.findUnique({ where: { id: previewId } });
  if (!preview) return;

-  let logs = (preview.logs || "") + text;
+  const stamped = stampLines(text);
+  let logs = (preview.logs || "") + stamped;
  if (Buffer.byteLength(logs, "utf8") > settings.logSizeLimitBytes) {
    const marker = "--- logs truncated ---\n";
    while (Buffer.byteLength(logs, "utf8") > settings.logSizeLimitBytes) {
@@ -51,7 +74,7 @@ export async function appendLog(previewId: number, text: string) {

  await prisma.preview.update({ where: { id: previewId }, data: { logs } });

-  broadcastLogUpdate(previewId, text);
+  broadcastLogUpdate(previewId, stamped);
 }

 const logSubscribers = new Map<number, Set<(text: string) => void>>();
@@ -94,7 +117,14 @@ export async function runDeploy(jobId: number) {
    if (isFirstDeploy) {
      await firstDeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle, cloneUrl);
    } else {
-      await redeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle);
+      const freshPreview = await prisma.preview.findUnique({ where: { id: preview.id } });
+      if (freshPreview?.instanceType && freshPreview.instanceType !== repoConfig.instanceType) {
+        await appendLog(preview.id, `[PP] Instance type changed from ${freshPreview.instanceType} to ${repoConfig.instanceType}; provisioning a fresh instance.\n`);
+        await stopPreview(preview.id, "STOPPED", "Instance type changed", true);
+        await firstDeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle, cloneUrl);
+      } else {
+        await redeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle);
+      }
    }

    await prisma.job.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: new Date() } });
@@ -168,7 +198,11 @@ async function firstDeploy(
    throw new Error("Concurrent instance limit reached");
  }

-  await updateStatus(previewId, "PROVISIONING", { commitSha, prNumber, prTitle });
+  // A restart from a STOPPED/IGNORED state reuses the same Preview row (see
+  // `/pp start` and stopped-branch push in webhook.ts), so its `logs` still hold
+  // the previous run's output. Clear them here so a fresh provision starts with a
+  // clean log instead of appending under the old EC2's build output.
+  await updateStatus(previewId, "PROVISIONING", { commitSha, prNumber, prTitle, logs: "", stopReason: null, stoppedAt: null });

  const commentBody = buildPrCommentBody({
    owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
@@ -207,7 +241,27 @@ async function firstDeploy(
    },
  });

-  await prisma.preview.update({ where: { id: previewId }, data: { instanceId } });
+  // Start a cost session for the new instance. If a prior session was left open
+  // (e.g. a failed deploy whose instance was never stopped before this restart),
+  // finalize its accrued cost first so it isn't lost when we overwrite the launch
+  // timestamp. See lib/cost.ts.
+  const priorCost = await prisma.preview.findUnique({
+    where: { id: previewId },
+    select: { accumulatedCostUsd: true, instanceLaunchedAt: true, instanceType: true },
+  });
+  let accumulatedCostUsd = priorCost?.accumulatedCostUsd ?? 0;
+  if (priorCost?.instanceLaunchedAt) {
+    accumulatedCostUsd += sessionCostUsd(priorCost.instanceLaunchedAt, new Date(), priorCost.instanceType);
+  }
+  await prisma.preview.update({
+    where: { id: previewId },
+    data: {
+      instanceId,
+      instanceType: repoConfig.instanceType,
+      instanceLaunchedAt: new Date(),
+      accumulatedCostUsd,
+    },
+  });
  await appendLog(previewId, `[PP] EC2 instance ${instanceId} launched. Waiting for running state...\n`);

  checkAbort(previewId);
@@ -223,25 +277,33 @@ async function firstDeploy(
    })
  );

-  await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH (as root)...\n`);
+  await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH (as ubuntu)...\n`);

  checkAbort(previewId);

-  // Ubuntu cloud images support command execution through their provisioned user.
-  const sshSession = await connectSsh(instanceIp, privateKey, 300_000);
+  // The `ubuntu` user accepts SSH almost immediately at boot, but cloud-init is
+  // still disabling apt timers and installing PP's base tools. Open a session
+  // just to wait for its completion marker before touching the instance,
+  // otherwise we can still hit apt-lock races below.
+  const bootstrapSession = await connectSsh(instanceIp, privateKey, 300_000);
+  activeSshSessions.set(previewId, bootstrapSession);
+  await appendLog(previewId, `[PP] Waiting for instance bootstrap to finish...\n`);
+  try {
+    checkAbort(previewId);
+    await waitForBootstrap(bootstrapSession);
+  } finally {
+    bootstrapSession.close();
+  }
+  checkAbort(previewId);
+
+  // Reconnect for the actual work so the marker-wait session stays short-lived
+  // and cannot carry stale state into the deploy steps.
+  const sshSession = await connectSsh(instanceIp, privateKey, 60_000);
  activeSshSessions.set(previewId, sshSession);

  try {
-    // Root SSH is enabled as the final bootstrap step, after the completion
-    // marker is written. A successful root connection is therefore the
-    // readiness signal; polling the same long-lived SSH connection can leave
-    // a completed instance stuck when its exec channel goes stale.
    await appendLog(previewId, `[PP] Bootstrap complete. Starting setup...\n`);

-    if (repoConfig.aptPackages.length > 0) {
-      await runSshStep(previewId, sshSession, `sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ${repoConfig.aptPackages.join(" ")}`);
-    }
-
    const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
    const parsedUrl = new URL(cloneUrl.startsWith("http") ? cloneUrl : `https://${cloneUrl}`);
    parsedUrl.username = "";
@@ -249,6 +311,12 @@ async function firstDeploy(
    const cleanCloneUrl = parsedUrl.toString();
    const authHeader = Buffer.from(`${user.giteaUsername || ""}:${giteaPat}`).toString("base64");

+    // We SSH in as the unprivileged `ubuntu` user, but /opt is root-owned, so
+    // an unprivileged clone into /opt/app fails with EACCES. Create the target
+    // directory up front and hand it to ubuntu so the clone (and every later
+    // step) can write without sudo.
+    await runSshStep(previewId, sshSession, `sudo mkdir -p /opt/app && sudo chown ubuntu:ubuntu /opt/app`);
+
    // Keep the PAT out of both preview logs and the URL. Gitea deployments can
    // reject userinfo URLs, while Git's per-command HTTP header works for both
    // private repositories and reverse proxies.
@@ -265,11 +333,21 @@ async function firstDeploy(
    }
    if (cloneResult.code !== 0) throw new Error(`git clone failed with exit code ${cloneResult.code}`);

-    await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`);
+    // The PR ref lives in the same private repo, so the fetch needs the same
+    // auth as the clone — otherwise git prompts for a username and aborts.
+    const authOption = giteaAuthOption(authHeader, giteaPat);
+    await runGitStep(
+      previewId,
+      sshSession,
+      `cd /opt/app && git ${authOption}fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`,
+      `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`,
+      authHeader,
+      giteaPat,
+    );

    await setupAndBuild(previewId, sshSession, repoConfig, commitSha, true);

-    await updateStatus(previewId, "RUNNING", { commitSha, instanceIp, port: repoConfig.port, lastActivityAt: new Date() });
+    await updateStatus(previewId, "RUNNING", { commitSha, instanceIp, port: repoConfig.port, lastActivityAt: new Date(), stopReason: null, stoppedAt: null });

    await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, commentId,
      buildPrCommentBody({
@@ -319,17 +397,45 @@ async function redeploy(
  try {
    if (repoConfig.useDockerCompose) {
      const composePath = repoConfig.composeFilePath || "docker-compose.yml";
-      await runSshStep(previewId, sshSession, `cd /opt/app && docker compose -f ${composePath} down 2>&1 || true`);
-    } else if (freshPreview.pid) {
-      await runSshStep(previewId, sshSession, `kill ${freshPreview.pid} 2>/dev/null || true; sleep 5; kill -9 ${freshPreview.pid} 2>/dev/null || true`);
+      await runSshStep(previewId, sshSession, `cd /opt/app && sudo docker compose -f ${shellQuote(composePath)} down 2>&1 || true`);
+    } else {
+      // Non-compose: stop the previous process. `pnpm start` (and npm/yarn) run
+      // the real server as a CHILD of the recorded PID, so killing just that PID
+      // can orphan the server on the app port; and a prior failed/timed-out
+      // deploy may not have recorded a PID at all. In both cases the next start
+      // dies with EADDRINUSE. Kill the recorded PID if we have one, then free the
+      // port itself as the authoritative backstop — `fuser -k` kills whatever is
+      // actually listening, clearing orphans regardless of how they were spawned.
+      const killPid = freshPreview.pid
+        ? `kill ${freshPreview.pid} 2>/dev/null || true; sleep 3; kill -9 ${freshPreview.pid} 2>/dev/null || true; `
+        : "";
+      await runSshStep(previewId, sshSession, `bash -c '${killPid}fuser -k ${repoConfig.port}/tcp 2>/dev/null || true; sleep 1; true'`);
    }

-    await runSshStep(previewId, sshSession, `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr && git reset --hard FETCH_HEAD`);
+    // Reuse the same per-command auth as firstDeploy; the checked-out repo has
+    // no persisted credentials, so every fetch must supply the header itself.
+    const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
+    const authHeader = Buffer.from(`${user.giteaUsername || ""}:${giteaPat}`).toString("base64");
+    const authOption = giteaAuthOption(authHeader, giteaPat);
+    // The working tree is already on the `pp-pr` branch from the first deploy,
+    // and git refuses to fetch directly into a checked-out branch ref of a
+    // non-bare repo ("Refusing to fetch into current branch"). Fetch into
+    // FETCH_HEAD instead (no branch-ref update), then force `pp-pr` to it with
+    // `checkout -B`, which also updates the working tree — equivalent to the old
+    // fetch+checkout+reset but without touching the live branch ref.
+    await runGitStep(
+      previewId,
+      sshSession,
+      `cd /opt/app && git ${authOption}fetch origin pull/${prNumber}/head && git checkout -B pp-pr FETCH_HEAD`,
+      `cd /opt/app && git fetch origin pull/${prNumber}/head && git checkout -B pp-pr FETCH_HEAD`,
+      authHeader,
+      giteaPat,
+    );

    await updateStatus(previewId, "BUILDING");
    await setupAndBuild(previewId, sshSession, repoConfig, commitSha, false);

-    await updateStatus(previewId, "RUNNING", { commitSha, lastActivityAt: new Date() });
+    await updateStatus(previewId, "RUNNING", { commitSha, lastActivityAt: new Date(), stopReason: null, stoppedAt: null });

    await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, newCommentId,
      buildPrCommentBody({
@@ -346,8 +452,21 @@ async function redeploy(

 const NVM_PREFIX = `export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh" 2>/dev/null;`;

-function withNvm(cmd: string): string {
-  return `bash -c '${NVM_PREFIX} ${cmd.replace(/'/g, `'"'"'`)}'`;
+function shellQuote(value: string): string {
+  return `'${value.replace(/'/g, `'"'"'`)}'`;
+}
+
+function bashLc(cmd: string): string {
+  return `bash -lc ${shellQuote(cmd)}`;
+}
+
+function usesNode(repoConfig: RepoConfig): boolean {
+  return normalizePreinstallTools((repoConfig as any).preinstallTools, repoConfig.useDockerCompose).includes("node");
+}
+
+function withRuntime(repoConfig: RepoConfig, cmd: string): string {
+  const prefix = usesNode(repoConfig) ? `${NVM_PREFIX} ` : "";
+  return bashLc(`${prefix}cd /opt/app && ${cmd}`);
 }

 async function setupAndBuild(
@@ -357,9 +476,7 @@ async function setupAndBuild(
  commitSha: string,
  isFirstProvision: boolean,
 ) {
-  // Detect and use Node version
-  const nvmCmd = `${NVM_PREFIX} if [ -f /opt/app/.nvmrc ]; then nvm install && nvm use; else nvm use default; fi`;
-  await runSshStep(previewId, sshSession, `bash -c '${nvmCmd}'`, false);
+  await ensurePreinstall(previewId, sshSession, repoConfig);

  // Write .env file safely using base64 to handle special chars and newlines
  const envVars = repoConfig.envVars as Record<string, string>;
@@ -371,7 +488,7 @@ async function setupAndBuild(

  if (isFirstProvision) {
    for (const cmd of repoConfig.setupCommands) {
-      await runSshStep(previewId, sshSession, withNvm(`cd /opt/app && ${cmd}`));
+      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
    }
  }

@@ -379,26 +496,146 @@ async function setupAndBuild(

  if (repoConfig.useDockerCompose) {
    const composePath = repoConfig.composeFilePath || "docker-compose.yml";
-    await runSshStep(previewId, sshSession, `cd /opt/app && docker compose -f ${composePath} up -d --build --force-recreate 2>&1`);
+    await runSshStep(previewId, sshSession, `cd /opt/app && sudo docker compose -f ${shellQuote(composePath)} up -d --build --force-recreate 2>&1`);
  } else {
+    // We SSH in and build as `ubuntu`, but a prior Docker-mode run (or a compose
+    // bind mount) can leave root-owned files under /opt/app — chiefly
+    // dependency folders — which makes package managers fail with EACCES when
+    // they try to rewrite them. Reclaim ownership before the build so switching
+    // a preview from docker-compose to a plain process doesn't wedge on
+    // permissions.
+    await runSshStep(previewId, sshSession, `sudo chown -R ubuntu:ubuntu /opt/app`);
+
    for (const cmd of repoConfig.buildCommands) {
-      await runSshStep(previewId, sshSession, withNvm(`cd /opt/app && ${cmd}`));
+      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
    }
    for (const cmd of repoConfig.postBuildCommands) {
-      await runSshStep(previewId, sshSession, withNvm(`cd /opt/app && ${cmd}`));
+      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
    }

-    if (repoConfig.runCommand) {
-      const startCmd = withNvm(`cd /opt/app && nohup ${repoConfig.runCommand} > /opt/app/pp.log 2>&1 & echo $!`);
-      const res = await runSshStep(previewId, sshSession, startCmd);
-      const pid = parseInt(res.stdout.trim(), 10);
-      if (!isNaN(pid)) {
-        await prisma.preview.update({ where: { id: previewId }, data: { pid } });
-      }
+    // Non-compose mode has no start step other than `runCommand`. If it's blank
+    // there is nothing to launch — fail loudly instead of marking the preview
+    // RUNNING with no process behind it (a misleading green preview).
+    const runCommand = (repoConfig.runCommand || "").trim();
+    if (!runCommand) {
+      throw new Error(
+        "No run command is configured for this repo (non-compose mode), so there is nothing to start. " +
+          'Set a Run Command in the repo config (e.g. "pnpm start"), or enable Docker Compose mode.',
+      );
+    }
+
+    // Detaching a long-lived server over SSH is subtle. A plain `nohup <cmd> &`
+    // leaves the process in the exec channel's process group, and sshd keeps the
+    // channel open until it exits — so the exec never returns and PP's 30s cap
+    // fires even though the app started fine. (Verified on a live instance: a
+    // bare `sleep` detaches instantly, but any long-running node/pnpm process
+    // hangs the channel.) `disown` removes the job from the shell so the channel
+    // can close immediately; `nohup` keeps it running past our session teardown;
+    // `< /dev/null` frees stdin. `echo $!` still yields the launched PID.
+    const startCmd = withRuntime(repoConfig, `{ nohup ${runCommand} > /opt/app/pp.log 2>&1 < /dev/null & } ; disown ; echo $!`);
+    const res = await runSshStep(previewId, sshSession, startCmd);
+    const pid = parseInt(res.stdout.trim(), 10);
+    if (!isNaN(pid)) {
+      await prisma.preview.update({ where: { id: previewId }, data: { pid } });
+      await healthCheck(previewId, sshSession, pid, repoConfig.port);
    }
  }
 }

+async function ensurePreinstall(previewId: number, sshSession: SshSession, repoConfig: RepoConfig) {
+  const tools = normalizePreinstallTools((repoConfig as any).preinstallTools, repoConfig.useDockerCompose);
+  if (tools.length === 0 && repoConfig.aptPackages.length === 0) {
+    await appendLog(previewId, "[PP] No preinstall options selected.\n");
+    return;
+  }
+
+  await appendLog(previewId, `[PP] Installing selected preinstall options: ${tools.length ? tools.join(", ") : "custom apt packages only"}.\n`);
+
+  const apt = `sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300`;
+  const aptPackages = new Set<string>();
+  if (tools.includes("python")) ["python3", "python3-pip", "python3-venv"].forEach(p => aptPackages.add(p));
+  if (tools.includes("go")) aptPackages.add("golang-go");
+  if (tools.includes("lua")) ["lua5.4", "luarocks"].forEach(p => aptPackages.add(p));
+  if (tools.includes("build-essential")) ["build-essential", "pkg-config"].forEach(p => aptPackages.add(p));
+  for (const pkg of normalizeAptPackages(repoConfig.aptPackages)) aptPackages.add(pkg);
+
+  if (aptPackages.size > 0) {
+    await runSshStep(previewId, sshSession, `${APT_WAIT}; ${apt} update && ${apt} install -y ${[...aptPackages].join(" ")}`);
+  }
+
+  if (tools.includes("docker")) {
+    await runSshStep(
+      previewId,
+      sshSession,
+      `${APT_WAIT}; if ! command -v docker >/dev/null 2>&1; then curl -fsSL https://get.docker.com | sudo sh; fi; ` +
+        `sudo systemctl enable docker && sudo systemctl start docker; ${APT_WAIT}; ${apt} update && ${apt} install -y docker-compose-plugin; sudo usermod -aG docker ubuntu`,
+    );
+  }
+
+  if (tools.includes("node")) {
+    const nodeVersion = String((repoConfig as any).nodeVersion || "lts/*");
+    await runSshStep(
+      previewId,
+      sshSession,
+      bashLc(
+        `if [ ! -s "$HOME/.nvm/nvm.sh" ]; then curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash; fi; ` +
+          `export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh"; ` +
+          `if [ -f /opt/app/.nvmrc ]; then cd /opt/app && nvm install && nvm use; ` +
+          `else nvm install ${shellQuote(nodeVersion)} && nvm alias default ${shellQuote(nodeVersion)} && nvm use default; fi`,
+      ),
+    );
+  }
+}
+
+// Wait for the freshly-launched app to come up. `nohup ... &` detaches the
+// process and routes its output to pp.log, so nothing from the server reaches
+// the preview logs — a clean start and an instant crash look identical. Poll
+// until either the process dies or *anything* accepts a TCP connection on the
+// app port (a bare connect, not an HTTP 2xx — the app owns the port either way),
+// then surface the first lines of pp.log so the startup is visible. Polls from
+// Node in short bursts because a single SSH exec is capped at 30s (see ssh.ts),
+// so a long in-shell loop would trip that timeout.
+const HEALTH_TIMEOUT_MS = 60_000;
+const HEALTH_INTERVAL_MS = 2_000;
+
+async function healthCheck(previewId: number, sshSession: SshSession, pid: number, port: number) {
+  await appendLog(previewId, `[PP] Waiting up to ${HEALTH_TIMEOUT_MS / 1000}s for the app to respond on port ${port}...\n`);
+
+  const start = Date.now();
+  let healthy = false;
+  let exited = false;
+
+  while (Date.now() - start < HEALTH_TIMEOUT_MS) {
+    checkAbortSession(sshSession);
+    // exit 2 = process gone, 0 = port accepted a connection, 1 = not yet.
+    // `/dev/tcp` is a bash builtin, so no nc/curl dependency; the connect runs
+    // in a subshell so its fd can't leak into later commands on this channel.
+    const probe = await sshSession.exec(
+      `bash -c 'kill -0 ${pid} 2>/dev/null || exit 2; (exec 3<>/dev/tcp/127.0.0.1/${port}) 2>/dev/null && exit 0 || exit 1'`,
+    );
+    if (probe.code === 0) { healthy = true; break; }
+    if (probe.code === 2) { exited = true; break; }
+    await sleep(HEALTH_INTERVAL_MS);
+  }
+
+  const tail = await sshSession.exec(`tail -n 40 /opt/app/pp.log 2>/dev/null`);
+  await appendLog(previewId, `--- server startup output (pp.log) ---\n`);
+  if (tail.stdout) await appendLog(previewId, tail.stdout);
+  await appendLog(previewId, `\n---\n`);
+
+  if (healthy) {
+    await appendLog(previewId, `[PP] Health check passed — port ${port} is accepting connections (pid ${pid}).\n`);
+  } else if (exited) {
+    throw new Error(`Server process ${pid} exited during startup — see pp.log output above.`);
+  } else {
+    throw new Error(`Health check timed out — nothing responded on port ${port} within ${HEALTH_TIMEOUT_MS / 1000}s. See pp.log output above.`);
+  }
+}
+
+function sleep(ms: number) {
+  return new Promise(r => setTimeout(r, ms));
+}
+
 async function runSshStep(previewId: number, sshSession: SshSession, command: string, throwOnFail = true) {
  checkAbortSession(sshSession);
  await appendLog(previewId, `$ ${command}\n`);
@@ -411,6 +648,37 @@ async function runSshStep(previewId: number, sshSession: SshSession, command: st
  return res;
 }

+// Like runSshStep, but for git commands that must carry the Gitea auth header.
+// The real command embeds the base64 credential inline (per-command, never
+// persisted to .git/config); `displayCommand` is the credential-free form we
+// echo to the logs, and any secret leaking into stdout/stderr is masked. This
+// mirrors the masking the initial clone already does.
+async function runGitStep(
+  previewId: number,
+  sshSession: SshSession,
+  realCommand: string,
+  displayCommand: string,
+  ...maskValues: string[]
+) {
+  checkAbortSession(sshSession);
+  await appendLog(previewId, `$ ${displayCommand}\n`);
+  const res = await sshSession.exec(realCommand);
+  const mask = (s: string) => maskValues.reduce((acc, m) => (m ? acc.split(m).join("****") : acc), s);
+  if (res.stdout) await appendLog(previewId, mask(res.stdout));
+  if (res.stderr) await appendLog(previewId, mask(res.stderr));
+  if (res.code !== 0) {
+    throw new Error(`Command failed with exit code ${res.code}: ${displayCommand}`);
+  }
+  return res;
+}
+
+// Builds the `-c http.extraHeader=...` git option that authenticates a single
+// git invocation against a private Gitea repo. Empty when there is no PAT
+// (public repos clone/fetch anonymously).
+function giteaAuthOption(authHeader: string, giteaPat: string): string {
+  return giteaPat ? `-c http.extraHeader='Authorization: Basic ${authHeader}' ` : "";
+}
+
 function checkAbortSession(session: SshSession) {
  if (session.aborted) throw new Error("ABORTED");
 }
@@ -429,7 +697,7 @@ function checkAbort(previewId: number) {
  }
 }

-export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED" = "STOPPED") {
+export async function stopPreview(previewId: number, status: "STOPPED" | "FAILED" = "STOPPED", reason = "Stopped", waitForCleanup = false) {
  const preview = await prisma.preview.findUnique({
    where: { id: previewId },
    include: { repoConfig: { include: { user: true } } },
@@ -441,34 +709,57 @@ export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED

  if (preview.instanceId) {
    const ec2 = makeEc2Client(user);
+    const instanceId = preview.instanceId;
    // Terminate instance first, then clean up key pair and security group
    try {
-      await terminateInstance(ec2, preview.instanceId);
+      await terminateInstance(ec2, instanceId);
    } catch (e) {
-      log.warn({ e, instanceId: preview.instanceId }, "Failed to terminate instance");
+      log.warn({ e, instanceId }, "Failed to terminate instance");
    }
    // Delete key pair immediately (doesn't depend on instance state)
    try {
      await deleteKeyPairAws(ec2, `pp-preview-${previewId}`);
    } catch {}
-    // Delete security group after a delay to allow instance ENI detachment
-    setTimeout(async () => {
+    // The security group can't be deleted until the instance's ENI is released,
+    // which only happens once the instance is fully terminated. Wait for that,
+    // then delete (deleteSecurityGroupAws also retries on DependencyViolation).
+    const cleanupSecurityGroup = async () => {
      try {
+        await waitForInstanceTerminated(ec2, instanceId);
        await deleteSecurityGroupAws(ec2, `pp-preview-${previewId}`);
      } catch (e) {
        log.warn({ e, previewId }, "Failed to delete security group after termination");
      }
-    }, 30_000);
+    };
+    if (waitForCleanup) await cleanupSecurityGroup();
+    else void cleanupSecurityGroup();
  }

+  // Close out the live cost session: fold the running instance's accrued cost
+  // into the finalized total and clear the launch timestamp so it stops billing.
+  const stoppedAt = new Date();
+  const finalCostUsd = computePreviewCostUsd(
+    {
+      accumulatedCostUsd: preview.accumulatedCostUsd,
+      instanceLaunchedAt: preview.instanceLaunchedAt,
+      instanceType: preview.instanceType,
+    },
+    stoppedAt,
+  );
+
+  const stopReason = reason.trim().slice(0, 128) || "Stopped";
+
  await prisma.preview.update({
    where: { id: previewId },
    data: {
-      status: reason,
-      stoppedAt: new Date(),
+      status,
+      stopReason,
+      stoppedAt,
      sshPrivateKey: null,
      sshKeyName: null,
      instanceId: null,
+      instanceLaunchedAt: null,
+      accumulatedCostUsd: finalCostUsd,
    },
  });

@@ -476,7 +767,7 @@ export async function stopPreview(previewId: number, reason: "STOPPED" | "FAILED
    owner: repoConfig.repoOwner,
    repo: repoConfig.repoName,
    prNumber: preview.prNumber,
-    status: "⚫ Stopped (inactivity timeout / PR closed / manual stop)",
+    status: `⚫ Stopped (${stopReason})`,
    commitSha: preview.commitSha,
    updatedAt: new Date(),
    ppBaseUrl: env.PP_BASE_URL,
@@ -59,6 +59,15 @@ export function makeStsClient(user: User): STSClient {

 export async function validateAwsCredentials(user: User): Promise<{ success: boolean; arn?: string; error?: string }> {
  try {
+    const accessKeyId = user.awsAccessKeyId ? decrypt(user.awsAccessKeyId) : "";
+    const region = user.awsRegion ?? "";
+    log.info("validateAwsCredentials: signer inputs", {
+      accessKeyId: JSON.stringify(accessKeyId),
+      accessKeyIdLen: accessKeyId.length,
+      accessKeyIdCharCodes: [...accessKeyId].map((c) => c.charCodeAt(0)),
+      region: JSON.stringify(region),
+      regionCharCodes: [...region].map((c) => c.charCodeAt(0)),
+    });
    const sts = makeStsClient(user);
    const res = await sts.send(new GetCallerIdentityCommand({}));
    return { success: true, arn: res.Arn };
@@ -118,20 +127,24 @@ const BOOTSTRAP_SCRIPT = `#!/bin/bash
 set -euo pipefail
 exec > >(tee -a /var/log/pp-bootstrap.log) 2>&1
 trap 'touch /var/lib/pp-bootstrap-failed' ERR
+
+# Ubuntu's apt-daily / unattended-upgrades timers fire on boot and hold the apt
+# locks, which makes apt-get die immediately ("Could not get lock") — not just
+# for our bootstrap, but for any user setup command that shells out to apt (e.g.
+# the get.docker.com install script). "mask --now" stops the units if they are
+# already running AND blocks them from being re-triggered for the life of this
+# ephemeral instance. Then wait out anything mid-flight before each apt-get.
+systemctl mask --now apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true
+apt_wait() {
+  for i in $(seq 1 150); do
+    fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 && sleep 2 || break
+  done
+}
+
+apt_wait
 apt-get update -y
-apt-get install -y curl git unzip build-essential
-
-# Docker
-curl -fsSL https://get.docker.com | sh
-systemctl enable docker
-systemctl start docker
-apt-get install -y docker-compose-plugin
-
-# PP deploys as the image's supported SSH user. Give it Docker access and its
-# own Node runtime rather than attempting to override Ubuntu's root SSH policy.
-usermod -aG docker ubuntu
-sudo -u ubuntu -H bash -lc 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash'
-sudo -u ubuntu -H bash -lc 'export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh"; nvm install --lts; nvm alias default lts/*'
+apt_wait
+apt-get install -y ca-certificates curl git unzip

 touch /var/lib/pp-bootstrap-done
 `;
@@ -188,6 +201,31 @@ export async function terminateInstance(ec2: EC2Client, instanceId: string): Pro
  await ec2.send(new TerminateInstancesCommand({ InstanceIds: [instanceId] }));
 }

+// Waits for an instance to reach the `terminated` state so its ENI is released
+// and the security group no longer has a dependent object. Resolves (rather than
+// throwing) on timeout so callers can still attempt SG deletion with retries.
+export async function waitForInstanceTerminated(
+  ec2: EC2Client,
+  instanceId: string,
+  timeoutMs = 180_000,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    try {
+      const res = await ec2.send(new DescribeInstancesCommand({
+        InstanceIds: [instanceId],
+      }));
+      const state = res.Reservations?.[0]?.Instances?.[0]?.State?.Name;
+      if (!state || state === "terminated") return;
+    } catch (e: any) {
+      // Instance record aged out / never existed — nothing left to wait on.
+      if (e.name === "InvalidInstanceID.NotFound") return;
+      throw e;
+    }
+    await sleep(5000);
+  }
+}
+
 export async function deleteKeyPairAws(ec2: EC2Client, keyName: string): Promise<void> {
  try {
    await ec2.send(new DeleteKeyPairCommand({ KeyName: keyName }));
@@ -202,9 +240,24 @@ export async function deleteSecurityGroupAws(ec2: EC2Client, groupName: string):
      Filters: [{ Name: "group-name", Values: [groupName] }],
    }));
    const groupId = describe.SecurityGroups?.[0]?.GroupId;
-    if (groupId) {
-      await sleep(5000);
-      await ec2.send(new DeleteSecurityGroupCommand({ GroupId: groupId }));
+    if (!groupId) return;
+
+    // The terminated instance's ENI can linger for a while after the instance
+    // itself is gone; DeleteSecurityGroup fails with DependencyViolation until
+    // that ENI is released. Retry with backoff instead of failing outright.
+    const maxAttempts = 12;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        await ec2.send(new DeleteSecurityGroupCommand({ GroupId: groupId }));
+        return;
+      } catch (e: any) {
+        if (e.name === "InvalidGroup.NotFound") return;
+        if (e.name === "DependencyViolation" && attempt < maxAttempts) {
+          await sleep(10_000);
+          continue;
+        }
+        throw e;
+      }
    }
  } catch (e) {
    log.warn({ e, groupName }, "Failed to delete security group");
@@ -23,13 +23,61 @@ export async function validateGiteaUrl(url: string): Promise<{ success: boolean;
  }
 }

+// Authenticates the PAT and confirms it belongs to the expected account. Gitea 1.19+
+// scopes tokens and does not expose granted scopes via the API, so we cannot pre-check
+// write:issue non-destructively — but authenticating here catches invalid/expired/wrong
+// tokens at setup instead of failing at deploy time.
+export async function validateGiteaToken(
+  url: string,
+  pat: string,
+  expectedUsername: string,
+): Promise<{ success: boolean; error?: string; login?: string }> {
+  try {
+    const res = await axios.get(`${url}/api/v1/user`, {
+      headers: { Authorization: `token ${pat}` },
+      timeout: 10000,
+    });
+    const login: string | undefined = res.data?.login;
+    if (expectedUsername && login && login.toLowerCase() !== expectedUsername.toLowerCase()) {
+      return {
+        success: false,
+        login,
+        error: `Token belongs to "${login}", not "${expectedUsername}". Use a token created by ${expectedUsername}.`,
+      };
+    }
+    return { success: true, login };
+  } catch (e: any) {
+    const status = e?.response?.status;
+    if (status === 401) return { success: false, error: "Token is invalid or expired." };
+    return { success: false, error: e?.message ?? "Token validation failed" };
+  }
+}
+
+// Translates opaque Gitea API errors into actionable messages. A 403 on a write almost
+// always means the PAT lacks the write scope for that resource.
+function giteaWriteError(e: any, action: string): Error {
+  if (e?.response?.status === 403) {
+    return new Error(
+      `Gitea denied ${action} (403 Forbidden). The Personal Access Token is missing write access. ` +
+        `Recreate it with Read and Write permission for the "issue" and "repository" scopes.`,
+    );
+  }
+  return e instanceof Error ? e : new Error(String(e));
+}
+
+// Lists only the repos the authenticated user owns or has access to (owned +
+// collaborator + org member). We deliberately use /user/repos rather than
+// /repos/search: for a Gitea site admin, /repos/search returns EVERY repo on the
+// instance, which would let admins see and manage repos they have no stake in.
+// /user/repos is user-scoped regardless of admin status. Note the shape differs:
+// /user/repos returns a bare array, not the /repos/search { ok, data } envelope.
 export async function fetchUserRepos(user: User): Promise<any[]> {
  const api = giteaApi(user);
  const repos: any[] = [];
  let page = 1;
  while (true) {
-    const res = await api.get(`/repos/search?limit=50&page=${page}`);
-    const data = res.data?.data ?? [];
+    const res = await api.get(`/user/repos?limit=50&page=${page}`);
+    const data = Array.isArray(res.data) ? res.data : [];
    if (data.length === 0) break;
    repos.push(...data);
    if (data.length < 50) break;
@@ -38,16 +86,34 @@ export async function fetchUserRepos(user: User): Promise<any[]> {
  return repos;
 }

+export async function fetchRepo(user: User, owner: string, repo: string): Promise<any> {
+  const api = giteaApi(user);
+  const res = await api.get(`/repos/${owner}/${repo}`);
+  return res.data;
+}
+
+// The canonical event set for every PP-managed webhook. Gitea splits PR events:
+// "pull_request" fires on opened/closed/reopened/edited, but a push of new
+// commits to the PR branch fires the SEPARATE "pull_request_sync" event.
+// Without it, the synchronize webhook is never delivered and previews never
+// redeploy on push. "issue_comment" carries the `/pp ...` commands.
+export const PP_WEBHOOK_EVENTS = ["pull_request", "pull_request_sync", "issue_comment"];
+
+// Human-readable label shown in Gitea's webhook list. Applied on creation and
+// backfilled onto older unnamed webhooks during boot reconciliation.
+export const PP_WEBHOOK_NAME = "PR Previews";
+
 export async function registerWebhook(user: User, owner: string, repo: string, webhookUrl: string, secret: string): Promise<number> {
  const api = giteaApi(user);
  const res = await api.post(`/repos/${owner}/${repo}/hooks`, {
    type: "gitea",
+    name: PP_WEBHOOK_NAME,
    config: {
      url: webhookUrl,
      secret,
      content_type: "json",
    },
-    events: ["pull_request", "issue_comment"],
+    events: PP_WEBHOOK_EVENTS,
    active: true,
  });
  return res.data.id;
@@ -60,26 +126,89 @@ export async function deleteWebhook(user: User, owner: string, repo: string, hoo

 export async function updateWebhookSecret(user: User, owner: string, repo: string, hookId: string, webhookUrl: string, newSecret: string): Promise<void> {
  const api = giteaApi(user);
+  const hook = await getWebhook(user, owner, repo, hookId);
+  if (!hook) throw new Error("Webhook no longer exists in Gitea");
+
+  const currentEvents: string[] = Array.isArray(hook.events) ? hook.events : [];
  await api.patch(`/repos/${owner}/${repo}/hooks/${hookId}`, {
+    name: PP_WEBHOOK_NAME,
    config: {
      url: webhookUrl,
      secret: newSecret,
      content_type: "json",
    },
-    events: ["pull_request", "issue_comment"],
+    events: [...new Set([...currentEvents, ...PP_WEBHOOK_EVENTS])],
    active: true,
  });
 }

+// Fetches a single webhook. Returns null on 404 (webhook deleted in Gitea while
+// PP still has its id on the RepoConfig).
+export async function getWebhook(user: User, owner: string, repo: string, hookId: string): Promise<any | null> {
+  const api = giteaApi(user);
+  try {
+    const res = await api.get(`/repos/${owner}/${repo}/hooks/${hookId}`);
+    return res.data;
+  } catch (e: any) {
+    if (e?.response?.status === 404) return null;
+    throw e;
+  }
+}
+
+export type WebhookReconcileResult =
+  | { status: "missing" }
+  | { status: "ok" }
+  | { status: "patched"; addedEvents: string[]; named: boolean; reactivated: boolean };
+
+// Brings an already-registered webhook up to the current PP standard without
+// tearing it down: unions in any missing events (chiefly "pull_request_sync"
+// for webhooks created before that fix), names it if it was left unnamed, and
+// re-activates it if disabled. User-added extra events are preserved. Patches
+// only when something actually differs, so re-runs are no-ops.
+export async function reconcileWebhook(user: User, owner: string, repo: string, hookId: string): Promise<WebhookReconcileResult> {
+  const hook = await getWebhook(user, owner, repo, hookId);
+  if (!hook) return { status: "missing" };
+
+  const currentEvents: string[] = Array.isArray(hook.events) ? hook.events : [];
+  const missingEvents = PP_WEBHOOK_EVENTS.filter(e => !currentEvents.includes(e));
+  const hasName = typeof hook.name === "string" && hook.name.trim().length > 0;
+  const inactive = hook.active === false;
+
+  if (missingEvents.length === 0 && hasName && !inactive) return { status: "ok" };
+
+  // Gitea's edit-hook API resets an omitted/empty `events` array to push-only,
+  // silently disabling every other trigger. So ANY patch must re-send the full
+  // desired event set (union of existing + PP events) and `active` — never a
+  // partial body that would wipe the triggers we depend on.
+  const patch: any = {
+    events: [...new Set([...currentEvents, ...PP_WEBHOOK_EVENTS])],
+    active: true,
+  };
+  if (!hasName) patch.name = PP_WEBHOOK_NAME;
+
+  const api = giteaApi(user);
+  await api.patch(`/repos/${owner}/${repo}/hooks/${hookId}`, patch);
+
+  return { status: "patched", addedEvents: missingEvents, named: !hasName, reactivated: inactive };
+}
+
 export async function postComment(user: User, owner: string, repo: string, issueNumber: number, body: string): Promise<number> {
  const api = giteaApi(user);
-  const res = await api.post(`/repos/${owner}/${repo}/issues/${issueNumber}/comments`, { body });
-  return res.data.id;
+  try {
+    const res = await api.post(`/repos/${owner}/${repo}/issues/${issueNumber}/comments`, { body });
+    return res.data.id;
+  } catch (e: any) {
+    throw giteaWriteError(e, "posting a PR comment");
+  }
 }

 export async function updateComment(user: User, owner: string, repo: string, commentId: number, body: string): Promise<void> {
  const api = giteaApi(user);
-  await api.patch(`/repos/${owner}/${repo}/issues/comments/${commentId}`, { body });
+  try {
+    await api.patch(`/repos/${owner}/${repo}/issues/comments/${commentId}`, { body });
+  } catch (e: any) {
+    throw giteaWriteError(e, "updating a PR comment");
+  }
 }

 export async function checkUserPermission(user: User, owner: string, repo: string, username: string): Promise<boolean> {
@@ -5,6 +5,11 @@ const log = createLogger("SSH");

 export interface SshSession {
  exec(command: string): Promise<{ stdout: string; stderr: string; code: number }>;
+  // Run a long-lived command (e.g. `tail -f`) and receive its stdout/stderr as
+  // it arrives. Unlike exec(), this never buffers or times out — the channel
+  // stays open until the returned handle's close() is called (or the connection
+  // ends). Used to stream live app logs off a running instance.
+  execStream(command: string, onData: (chunk: string) => void): { close(): void };
  close(): void;
  aborted: boolean;
  abort(): void;
@@ -28,6 +33,26 @@ export async function connectSsh(host: string, privateKey: string, maxWaitMs = 3
          if (aborted) throw new Error("SSH session aborted");
          return execOnConn(conn, command);
        },
+        execStream(command: string, onData: (chunk: string) => void) {
+          let stream: any = null;
+          let closed = false;
+          conn.exec(command, (err, s) => {
+            if (err) {
+              if (!closed) onData(`[app-logs] stream error: ${err.message}\n`);
+              return;
+            }
+            if (closed) { try { s.close(); } catch {} return; }
+            stream = s;
+            s.on("data", (d: Buffer) => onData(d.toString()));
+            s.stderr.on("data", (d: Buffer) => onData(d.toString()));
+          });
+          return {
+            close() {
+              closed = true;
+              try { stream?.close(); } catch {}
+            },
+          };
+        },
        close() {
          try { conn.end(); } catch {}
        },
@@ -95,14 +120,37 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st
  });
 }

+// The `ubuntu` user accepts SSH almost immediately at boot — long before the
+// cloud-init UserData bootstrap has finished disabling apt-daily and installing
+// the small base dependency set PP needs before SSH-driven setup begins.
+// Proceeding early can still cause apt-lock failures in the deploy steps. Block until the bootstrap writes its
+// done marker; bail out fast (with the tail of its log) if it writes the failure
+// marker instead. Uses short one-shot execs so a stale channel just retries.
 export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise<void> {
  const start = Date.now();
  while (Date.now() - start < maxWaitMs) {
+    if (session.aborted) throw new Error("SSH session aborted");
    try {
-      const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
-      if (res.stdout.trim() === "done") return;
-    } catch {
-      // transient SSH exec error (e.g. timeout, channel reset) — retry
+      const res = await session.exec(
+        "if [ -f /var/lib/pp-bootstrap-failed ]; then echo failed; " +
+        "elif [ -f /var/lib/pp-bootstrap-done ]; then echo done; else echo waiting; fi"
+      );
+      const state = res.stdout.trim();
+      if (state === "done") return;
+      if (state === "failed") {
+        let tail = "";
+        try {
+          const logRes = await session.exec("tail -n 30 /var/log/pp-bootstrap.log 2>/dev/null || true");
+          tail = logRes.stdout.trim();
+        } catch {
+          // best effort — the failure marker alone is enough to abort
+        }
+        throw new Error(`EC2 bootstrap failed${tail ? `:\n${tail}` : ""}`);
+      }
+    } catch (e: any) {
+      // A bootstrap failure we detected above must propagate; only swallow
+      // transient SSH exec errors (timeout, channel reset) and retry.
+      if (e?.message?.startsWith("EC2 bootstrap failed")) throw e;
    }
    await sleep(10_000);
  }
@@ -0,0 +1,48 @@
+import { prisma } from "../lib/db";
+import { createLogger } from "../lib/logger";
+import { reconcileWebhook } from "./gitea";
+
+const log = createLogger("WEBHOOK_RECONCILE");
+
+// On boot, bring every PP-managed Gitea webhook up to the current standard so
+// existing installs self-heal without the operator having to disable/re-enable
+// each repo. Chiefly backfills the "pull_request_sync" event (added after these
+// webhooks were first registered — without it pushes to a PR branch never
+// redeploy) and names any webhook left unnamed. Errors are per-repo so one bad
+// PAT or deleted repo can't abort the whole pass.
+export async function runWebhookReconciliation() {
+  log.info("Reconciling Gitea webhooks");
+
+  const configs = await prisma.repoConfig.findMany({
+    where: { isEnabled: true, giteaWebhookId: { not: null } },
+    include: { user: true },
+  });
+
+  let patched = 0;
+  let named = 0;
+  let missing = 0;
+
+  for (const config of configs) {
+    const { user, repoOwner, repoName, giteaWebhookId } = config;
+    if (!user?.giteaInstanceUrl || !user?.giteaPAT || !giteaWebhookId) continue;
+
+    try {
+      const result = await reconcileWebhook(user as any, repoOwner, repoName, giteaWebhookId);
+      if (result.status === "missing") {
+        missing++;
+        log.warn({ repo: `${repoOwner}/${repoName}`, giteaWebhookId }, "Webhook no longer exists in Gitea");
+      } else if (result.status === "patched") {
+        patched++;
+        if (result.named) named++;
+        log.info(
+          { repo: `${repoOwner}/${repoName}`, addedEvents: result.addedEvents, named: result.named, reactivated: result.reactivated },
+          "Webhook reconciled",
+        );
+      }
+    } catch (e) {
+      log.warn({ e, repo: `${repoOwner}/${repoName}`, userId: user.id }, "Failed to reconcile webhook");
+    }
+  }
+
+  log.info({ total: configs.length, patched, named, missing }, "Webhook reconciliation complete");
+}
@@ -6,8 +6,8 @@ import { getAdminSettings } from "../lib/adminSettings";
 const log = createLogger("CRON");

 export function startCronWorkers() {
-  // Inactivity check every 30 minutes
-  cron.schedule("*/30 * * * *", async () => {
+  // Inactivity check every 5 minutes
+  cron.schedule("*/5 * * * *", async () => {
    try {
      await checkInactivity();
    } catch (e) {
@@ -24,6 +24,10 @@ export function startCronWorkers() {
    }
  });

+  // Run an inactivity check immediately on startup so previews that went idle
+  // while PP was down aren't left waiting for the next scheduled tick.
+  checkInactivity().catch((e) => log.error({ e }, "Startup inactivity check error"));
+
  log.info("Cron workers started");
 }

@@ -76,8 +76,14 @@ async function processJob(job: { id: number; type: string; previewId: number | n
      await runDeploy(job.id);
    } else if (job.type === "STOP" || job.type === "INACTIVITY_STOP") {
      if (job.previewId) {
+        const payload = job.payload && typeof job.payload === "object" ? job.payload : {};
+        const reason = typeof payload.reason === "string" && payload.reason.trim()
+          ? payload.reason.trim()
+          : job.type === "INACTIVITY_STOP"
+            ? "Inactivity timeout"
+            : "Stopped";
        await prisma.job.update({ where: { id: job.id }, data: { status: "RUNNING", startedAt: new Date() } });
-        await stopPreview(job.previewId);
+        await stopPreview(job.previewId, "STOPPED", reason);
        await prisma.job.update({ where: { id: job.id }, data: { status: "DONE", finishedAt: new Date() } });
      }
    }