import { prisma } from "../lib/db";
import { createLogger } from "../lib/logger";
import { decrypt, encrypt } from "../lib/encryption";
import { getAdminSettings } from "../lib/adminSettings";
import { env } from "../lib/env";
import {
  makeEc2Client,
  createKeyPairAndGetPrivateKey,
  createPreviewSecurityGroup,
  launchInstance,
  waitForInstanceRunning,
  terminateInstance,
  waitForInstanceTerminated,
  deleteKeyPairAws,
  deleteSecurityGroupAws,
} from "./ec2";
import { connectSsh, waitForBootstrap, type SshSession } from "./ssh";
import { computePreviewCostUsd, sessionCostUsd } from "../lib/cost";
import { normalizeAptPackages, normalizePreinstallTools } from "../lib/preinstall";
import {
  buildPrCommentBody,
  postComment,
  updateComment,
} from "./gitea";
import type { Preview, RepoConfig, User } from "@prisma/client";

const log = createLogger("DEPLOY");

// Block until Ubuntu's apt-daily / unattended-upgrades timers release every apt
// lock, so a following `apt-get update`/`install` doesn't die with "Could not
// get lock". Polls all four lock files for up to 300s (150 * 2s). `fuser`
// returns 0 while a file is in use, non-zero once it's free.
const APT_WAIT =
  `sudo bash -c 'for i in $(seq 1 150); do ` +
  `fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1 ` +
  `&& sleep 2 || break; done'`;

const activeSshSessions = new Map<number, SshSession>();

export function abortJobForPreview(previewId: number) {
  const session = activeSshSessions.get(previewId);
  if (session) {
    session.abort();
    activeSshSessions.delete(previewId);
  }
}

// Prefix every non-empty line of a log chunk with a local-time `[HH:MM:SS]`
// stamp. SSH execs buffer their whole output and hand it to appendLog at once,
// so a command's lines share the stamp of the moment it completed — the useful
// signal is *when each step happened*, which this captures. Blank lines (e.g.
// the `\n---\n` separators) are left untouched so the log keeps its spacing.
function stampLines(text: string): string {
  const stamp = `[${new Date().toTimeString().slice(0, 8)}] `;
  return text.replace(/^(?=.)/gm, stamp);
}

export async function appendLog(previewId: number, text: string) {
  const settings = await getAdminSettings();
  const preview = await prisma.preview.findUnique({ where: { id: previewId } });
  if (!preview) return;

  const stamped = stampLines(text);
  let logs = (preview.logs || "") + stamped;
  if (Buffer.byteLength(logs, "utf8") > settings.logSizeLimitBytes) {
    const marker = "--- logs truncated ---\n";
    while (Buffer.byteLength(logs, "utf8") > settings.logSizeLimitBytes) {
      const nl = logs.indexOf("\n");
      if (nl === -1) break;
      logs = logs.slice(nl + 1);
    }
    logs = marker + logs;
  }

  await prisma.preview.update({ where: { id: previewId }, data: { logs } });

  broadcastLogUpdate(previewId, stamped);
}

const logSubscribers = new Map<number, Set<(text: string) => void>>();

export function subscribeToLogs(previewId: number, cb: (text: string) => void): () => void {
  if (!logSubscribers.has(previewId)) logSubscribers.set(previewId, new Set());
  logSubscribers.get(previewId)!.add(cb);
  return () => logSubscribers.get(previewId)?.delete(cb);
}

function broadcastLogUpdate(previewId: number, text: string) {
  logSubscribers.get(previewId)?.forEach(cb => cb(text));
}

async function updateStatus(previewId: number, status: Preview["status"], extra: Partial<Preview> = {}) {
  await prisma.preview.update({ where: { id: previewId }, data: { status, ...extra } });
}

export async function runDeploy(jobId: number) {
  const job = await prisma.job.findUnique({ where: { id: jobId }, include: { preview: { include: { repoConfig: { include: { user: true } } } } } });
  if (!job || !job.preview) {
    log.error({ jobId }, "Job or preview not found");
    return;
  }

  // Clear any stale abort signal left over from aborting the previous job for this preview.
  // The new job should not be cancelled by a signal meant for the old one.
  if (job.preview.id) abortSignals.delete(job.preview.id);

  await prisma.job.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: new Date() } });

  const preview = job.preview as any;
  const repoConfig: RepoConfig = preview.repoConfig;
  const user: User = (preview.repoConfig as any).user;

  const payload = job.payload as any;
  const { commitSha, prNumber, prTitle, cloneUrl, isFirstDeploy } = payload;

  try {
    if (isFirstDeploy) {
      await firstDeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle, cloneUrl);
    } else {
      const freshPreview = await prisma.preview.findUnique({ where: { id: preview.id } });
      if (freshPreview?.instanceType && freshPreview.instanceType !== repoConfig.instanceType) {
        await appendLog(preview.id, `[PP] Instance type changed from ${freshPreview.instanceType} to ${repoConfig.instanceType}; provisioning a fresh instance.\n`);
        await stopPreview(preview.id, "STOPPED", "Instance type changed", true);
        await firstDeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle, cloneUrl);
      } else {
        await redeploy(jobId, preview, repoConfig, user, commitSha, prNumber, prTitle);
      }
    }

    await prisma.job.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: new Date() } });
  } catch (e: any) {
    if (e.message === "ABORTED") {
      log.info({ jobId, previewId: preview.id }, "Job aborted");
      await prisma.job.update({ where: { id: jobId }, data: { status: "FAILED", error: "Aborted by newer deploy", finishedAt: new Date() } });
      return;
    }
    log.error({ e, jobId, previewId: preview.id }, "Deploy failed");
    await prisma.job.update({ where: { id: jobId }, data: { status: "FAILED", error: e.message, finishedAt: new Date() } });

    const freshPreview = await prisma.preview.findUnique({ where: { id: preview.id } });
    if (!freshPreview) return;

    const lastLines = (freshPreview.logs || "").split("\n").slice(-10).join("\n");
    await updateStatus(preview.id, "FAILED");

    const failBody = buildPrCommentBody({
      owner: repoConfig.repoOwner,
      repo: repoConfig.repoName,
      prNumber: freshPreview.prNumber,
      status: `🔴 Failed — last log lines:\n\`\`\`\n${lastLines}\n\`\`\``,
      commitSha: freshPreview.commitSha,
      updatedAt: new Date(),
      ppBaseUrl: env.PP_BASE_URL,
    });

    if (freshPreview.giteaCommentId) {
      try {
        await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, freshPreview.giteaCommentId, failBody);
      } catch {}
    }
  }
}

async function firstDeploy(
  jobId: number,
  preview: Preview & { repoConfig: RepoConfig },
  repoConfig: RepoConfig,
  user: User,
  commitSha: string,
  prNumber: number,
  prTitle: string,
  cloneUrl: string,
) {
  const settings = await getAdminSettings();
  const ec2 = makeEc2Client(user);
  const previewId = preview.id;

  const activeCount = await prisma.preview.count({
    where: {
      repoConfig: { userId: user.id },
      status: { in: ["PROVISIONING", "BUILDING", "RUNNING"] },
      id: { not: previewId },
    },
  });

  if (activeCount >= settings.maxConcurrentInstancesPerUser) {
    const body = buildPrCommentBody({
      owner: repoConfig.repoOwner,
      repo: repoConfig.repoName,
      prNumber,
      status: `🔴 Cannot provision preview — concurrent instance limit (${settings.maxConcurrentInstancesPerUser}) reached.`,
      commitSha,
      updatedAt: new Date(),
      ppBaseUrl: env.PP_BASE_URL,
    });
    const commentId = await postComment(user, repoConfig.repoOwner, repoConfig.repoName, prNumber, body);
    await prisma.preview.update({ where: { id: previewId }, data: { giteaCommentId: commentId } });
    throw new Error("Concurrent instance limit reached");
  }

  // A restart from a STOPPED/IGNORED state reuses the same Preview row (see
  // `/pp start` and stopped-branch push in webhook.ts), so its `logs` still hold
  // the previous run's output. Clear them here so a fresh provision starts with a
  // clean log instead of appending under the old EC2's build output.
  await updateStatus(previewId, "PROVISIONING", { commitSha, prNumber, prTitle, logs: "", stopReason: null, stoppedAt: null });

  const commentBody = buildPrCommentBody({
    owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
    status: "🟡 Provisioning EC2 instance...",
    commitSha, updatedAt: new Date(), ppBaseUrl: env.PP_BASE_URL,
  });
  const commentId = await postComment(user, repoConfig.repoOwner, repoConfig.repoName, prNumber, commentBody);
  await prisma.preview.update({ where: { id: previewId }, data: { giteaCommentId: commentId } });

  checkAbort(previewId);

  const keyName = `pp-preview-${previewId}`;
  const { privateKey } = await createKeyPairAndGetPrivateKey(ec2, keyName);
  const encPrivateKey = encrypt(privateKey);

  await prisma.preview.update({ where: { id: previewId }, data: { sshPrivateKey: encPrivateKey, sshKeyName: keyName } });

  checkAbort(previewId);

  const sgName = `pp-preview-${previewId}`;
  const securityGroupId = await createPreviewSecurityGroup(ec2, sgName, repoConfig.port);

  checkAbort(previewId);

  const instanceId = await launchInstance({
    ec2, region: user.awsRegion!,
    instanceType: repoConfig.instanceType,
    keyName,
    securityGroupId,
    tags: {
      "pp:managed": "true",
      "pp:userId": String(user.id),
      "pp:repo": `${repoConfig.repoOwner}/${repoConfig.repoName}`,
      "pp:prNumber": String(prNumber),
      "pp:previewId": String(previewId),
    },
  });

  // Start a cost session for the new instance. If a prior session was left open
  // (e.g. a failed deploy whose instance was never stopped before this restart),
  // finalize its accrued cost first so it isn't lost when we overwrite the launch
  // timestamp. See lib/cost.ts.
  const priorCost = await prisma.preview.findUnique({
    where: { id: previewId },
    select: { accumulatedCostUsd: true, instanceLaunchedAt: true, instanceType: true },
  });
  let accumulatedCostUsd = priorCost?.accumulatedCostUsd ?? 0;
  if (priorCost?.instanceLaunchedAt) {
    accumulatedCostUsd += sessionCostUsd(priorCost.instanceLaunchedAt, new Date(), priorCost.instanceType);
  }
  await prisma.preview.update({
    where: { id: previewId },
    data: {
      instanceId,
      instanceType: repoConfig.instanceType,
      instanceLaunchedAt: new Date(),
      accumulatedCostUsd,
    },
  });
  await appendLog(previewId, `[PP] EC2 instance ${instanceId} launched. Waiting for running state...\n`);

  checkAbort(previewId);

  const instanceIp = await waitForInstanceRunning(ec2, instanceId);
  await prisma.preview.update({ where: { id: previewId }, data: { instanceIp, port: repoConfig.port } });

  await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, commentId,
    buildPrCommentBody({
      owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
      status: `🟡 Building... (EC2 ready at ${instanceIp})`,
      commitSha, updatedAt: new Date(), ppBaseUrl: env.PP_BASE_URL,
    })
  );

  await appendLog(previewId, `[PP] Instance running at ${instanceIp}. Waiting for SSH (as ubuntu)...\n`);

  checkAbort(previewId);

  // The `ubuntu` user accepts SSH almost immediately at boot, but cloud-init is
  // still disabling apt timers and installing PP's base tools. Open a session
  // just to wait for its completion marker before touching the instance,
  // otherwise we can still hit apt-lock races below.
  const bootstrapSession = await connectSsh(instanceIp, privateKey, 300_000);
  activeSshSessions.set(previewId, bootstrapSession);
  await appendLog(previewId, `[PP] Waiting for instance bootstrap to finish...\n`);
  try {
    checkAbort(previewId);
    await waitForBootstrap(bootstrapSession);
  } finally {
    bootstrapSession.close();
  }
  checkAbort(previewId);

  // Reconnect for the actual work so the marker-wait session stays short-lived
  // and cannot carry stale state into the deploy steps.
  const sshSession = await connectSsh(instanceIp, privateKey, 60_000);
  activeSshSessions.set(previewId, sshSession);

  try {
    await appendLog(previewId, `[PP] Bootstrap complete. Starting setup...\n`);

    const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
    const parsedUrl = new URL(cloneUrl.startsWith("http") ? cloneUrl : `https://${cloneUrl}`);
    parsedUrl.username = "";
    parsedUrl.password = "";
    const cleanCloneUrl = parsedUrl.toString();
    const authHeader = Buffer.from(`${user.giteaUsername || ""}:${giteaPat}`).toString("base64");

    // We SSH in as the unprivileged `ubuntu` user, but /opt is root-owned, so
    // an unprivileged clone into /opt/app fails with EACCES. Create the target
    // directory up front and hand it to ubuntu so the clone (and every later
    // step) can write without sudo.
    await runSshStep(previewId, sshSession, `sudo mkdir -p /opt/app && sudo chown ubuntu:ubuntu /opt/app`);

    // Keep the PAT out of both preview logs and the URL. Gitea deployments can
    // reject userinfo URLs, while Git's per-command HTTP header works for both
    // private repositories and reverse proxies.
    await appendLog(previewId, `$ git clone '${cleanCloneUrl}' /opt/app\n`);
    const cloneCommand = giteaPat
      ? `git -c http.extraHeader='Authorization: Basic ${authHeader}' clone '${cleanCloneUrl}' /opt/app`
      : `git clone '${cleanCloneUrl}' /opt/app`;
    const cloneResult = await sshSession.exec(cloneCommand);
    if (cloneResult.stdout) await appendLog(previewId, cloneResult.stdout);
    if (cloneResult.stderr) {
      // Mask PAT in stderr output too
      const maskedStderr = cloneResult.stderr.replace(giteaPat, "****").replace(authHeader, "****");
      await appendLog(previewId, maskedStderr);
    }
    if (cloneResult.code !== 0) throw new Error(`git clone failed with exit code ${cloneResult.code}`);

    // The PR ref lives in the same private repo, so the fetch needs the same
    // auth as the clone — otherwise git prompts for a username and aborts.
    const authOption = giteaAuthOption(authHeader, giteaPat);
    await runGitStep(
      previewId,
      sshSession,
      `cd /opt/app && git ${authOption}fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`,
      `cd /opt/app && git fetch origin pull/${prNumber}/head:pp-pr && git checkout pp-pr`,
      authHeader,
      giteaPat,
    );

    await setupAndBuild(previewId, sshSession, repoConfig, commitSha, true);

    await updateStatus(previewId, "RUNNING", { commitSha, instanceIp, port: repoConfig.port, lastActivityAt: new Date(), stopReason: null, stoppedAt: null });

    await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, commentId,
      buildPrCommentBody({
        owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
        status: `🟢 Live at http://${instanceIp}:${repoConfig.port}`,
        commitSha, updatedAt: new Date(), ppBaseUrl: env.PP_BASE_URL,
      })
    );
  } finally {
    sshSession.close();
    activeSshSessions.delete(previewId);
  }
}

async function redeploy(
  jobId: number,
  preview: Preview & { repoConfig: RepoConfig },
  repoConfig: RepoConfig,
  user: User,
  commitSha: string,
  prNumber: number,
  prTitle: string,
) {
  const previewId = preview.id;

  // Always use fresh data from DB for connection details
  const freshPreview = await prisma.preview.findUnique({ where: { id: previewId } });
  if (!freshPreview?.instanceIp || !freshPreview?.sshPrivateKey) {
    throw new Error("No active instance found for redeploy — cannot SSH in");
  }
  const instanceIp = freshPreview.instanceIp;
  const privateKey = decrypt(freshPreview.sshPrivateKey);

  const sshSession = await connectSsh(instanceIp, privateKey, 60_000);
  activeSshSessions.set(previewId, sshSession);

  await appendLog(previewId, `\n--- Redeploy: ${commitSha} ---\n`);

  const commentBody = buildPrCommentBody({
    owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
    status: `🟡 Building... (EC2 at ${instanceIp})`,
    commitSha, updatedAt: new Date(), ppBaseUrl: env.PP_BASE_URL,
  });
  const newCommentId = await postComment(user, repoConfig.repoOwner, repoConfig.repoName, prNumber, commentBody);
  await prisma.preview.update({ where: { id: previewId }, data: { giteaCommentId: newCommentId, commitSha } });

  try {
    if (repoConfig.useDockerCompose) {
      const composePath = repoConfig.composeFilePath || "docker-compose.yml";
      await runSshStep(previewId, sshSession, `cd /opt/app && sudo docker compose -f ${shellQuote(composePath)} down 2>&1 || true`);
    } else {
      // Non-compose: stop the previous process. `pnpm start` (and npm/yarn) run
      // the real server as a CHILD of the recorded PID, so killing just that PID
      // can orphan the server on the app port; and a prior failed/timed-out
      // deploy may not have recorded a PID at all. In both cases the next start
      // dies with EADDRINUSE. Kill the recorded PID if we have one, then free the
      // port itself as the authoritative backstop — `fuser -k` kills whatever is
      // actually listening, clearing orphans regardless of how they were spawned.
      const killPid = freshPreview.pid
        ? `kill ${freshPreview.pid} 2>/dev/null || true; sleep 3; kill -9 ${freshPreview.pid} 2>/dev/null || true; `
        : "";
      await runSshStep(previewId, sshSession, `bash -c '${killPid}fuser -k ${repoConfig.port}/tcp 2>/dev/null || true; sleep 1; true'`);
    }

    // Reuse the same per-command auth as firstDeploy; the checked-out repo has
    // no persisted credentials, so every fetch must supply the header itself.
    const giteaPat = user.giteaPAT ? decrypt(user.giteaPAT) : "";
    const authHeader = Buffer.from(`${user.giteaUsername || ""}:${giteaPat}`).toString("base64");
    const authOption = giteaAuthOption(authHeader, giteaPat);
    // The working tree is already on the `pp-pr` branch from the first deploy,
    // and git refuses to fetch directly into a checked-out branch ref of a
    // non-bare repo ("Refusing to fetch into current branch"). Fetch into
    // FETCH_HEAD instead (no branch-ref update), then force `pp-pr` to it with
    // `checkout -B`, which also updates the working tree — equivalent to the old
    // fetch+checkout+reset but without touching the live branch ref.
    await runGitStep(
      previewId,
      sshSession,
      `cd /opt/app && git ${authOption}fetch origin pull/${prNumber}/head && git checkout -B pp-pr FETCH_HEAD`,
      `cd /opt/app && git fetch origin pull/${prNumber}/head && git checkout -B pp-pr FETCH_HEAD`,
      authHeader,
      giteaPat,
    );

    await updateStatus(previewId, "BUILDING");
    await setupAndBuild(previewId, sshSession, repoConfig, commitSha, false);

    await updateStatus(previewId, "RUNNING", { commitSha, lastActivityAt: new Date(), stopReason: null, stoppedAt: null });

    await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, newCommentId,
      buildPrCommentBody({
        owner: repoConfig.repoOwner, repo: repoConfig.repoName, prNumber,
        status: `🟢 Live at http://${instanceIp}:${repoConfig.port}`,
        commitSha, updatedAt: new Date(), ppBaseUrl: env.PP_BASE_URL,
      })
    );
  } finally {
    sshSession.close();
    activeSshSessions.delete(previewId);
  }
}

const NVM_PREFIX = `export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh" 2>/dev/null;`;

function shellQuote(value: string): string {
  return `'${value.replace(/'/g, `'"'"'`)}'`;
}

function bashLc(cmd: string): string {
  return `bash -lc ${shellQuote(cmd)}`;
}

function usesNode(repoConfig: RepoConfig): boolean {
  return normalizePreinstallTools((repoConfig as any).preinstallTools, repoConfig.useDockerCompose).includes("node");
}

function withRuntime(repoConfig: RepoConfig, cmd: string): string {
  const prefix = usesNode(repoConfig) ? `${NVM_PREFIX} ` : "";
  return bashLc(`${prefix}cd /opt/app && ${cmd}`);
}

async function setupAndBuild(
  previewId: number,
  sshSession: SshSession,
  repoConfig: RepoConfig,
  commitSha: string,
  isFirstProvision: boolean,
) {
  await ensurePreinstall(previewId, sshSession, repoConfig);

  // Write .env file safely using base64 to handle special chars and newlines
  const envVars = repoConfig.envVars as Record<string, string>;
  const envContent = Object.entries(envVars).map(([k, v]) => `${k}=${v}`).join("\n") + "\n";
  const envB64 = Buffer.from(envContent).toString("base64");
  await appendLog(previewId, `$ echo '<base64 .env> | base64 -d > /opt/app/.env'\n`);
  const envResult = await sshSession.exec(`echo '${envB64}' | base64 -d > /opt/app/.env`);
  if (envResult.code !== 0) throw new Error(".env write failed");

  if (isFirstProvision) {
    for (const cmd of repoConfig.setupCommands) {
      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
    }
  }

  await updateStatus(previewId, "BUILDING");

  if (repoConfig.useDockerCompose) {
    const composePath = repoConfig.composeFilePath || "docker-compose.yml";
    await runSshStep(previewId, sshSession, `cd /opt/app && sudo docker compose -f ${shellQuote(composePath)} up -d --build --force-recreate 2>&1`);
  } else {
    // We SSH in and build as `ubuntu`, but a prior Docker-mode run (or a compose
    // bind mount) can leave root-owned files under /opt/app — chiefly
    // dependency folders — which makes package managers fail with EACCES when
    // they try to rewrite them. Reclaim ownership before the build so switching
    // a preview from docker-compose to a plain process doesn't wedge on
    // permissions.
    await runSshStep(previewId, sshSession, `sudo chown -R ubuntu:ubuntu /opt/app`);

    for (const cmd of repoConfig.buildCommands) {
      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
    }
    for (const cmd of repoConfig.postBuildCommands) {
      await runSshStep(previewId, sshSession, withRuntime(repoConfig, cmd));
    }

    // Non-compose mode has no start step other than `runCommand`. If it's blank
    // there is nothing to launch — fail loudly instead of marking the preview
    // RUNNING with no process behind it (a misleading green preview).
    const runCommand = (repoConfig.runCommand || "").trim();
    if (!runCommand) {
      throw new Error(
        "No run command is configured for this repo (non-compose mode), so there is nothing to start. " +
          'Set a Run Command in the repo config (e.g. "pnpm start"), or enable Docker Compose mode.',
      );
    }

    // Detaching a long-lived server over SSH is subtle. A plain `nohup <cmd> &`
    // leaves the process in the exec channel's process group, and sshd keeps the
    // channel open until it exits — so the exec never returns and PP's 30s cap
    // fires even though the app started fine. (Verified on a live instance: a
    // bare `sleep` detaches instantly, but any long-running node/pnpm process
    // hangs the channel.) `disown` removes the job from the shell so the channel
    // can close immediately; `nohup` keeps it running past our session teardown;
    // `< /dev/null` frees stdin. `echo $!` still yields the launched PID.
    const startCmd = withRuntime(repoConfig, `{ nohup ${runCommand} > /opt/app/pp.log 2>&1 < /dev/null & } ; disown ; echo $!`);
    const res = await runSshStep(previewId, sshSession, startCmd);
    const pid = parseInt(res.stdout.trim(), 10);
    if (!isNaN(pid)) {
      await prisma.preview.update({ where: { id: previewId }, data: { pid } });
      await healthCheck(previewId, sshSession, pid, repoConfig.port);
    }
  }
}

async function ensurePreinstall(previewId: number, sshSession: SshSession, repoConfig: RepoConfig) {
  const tools = normalizePreinstallTools((repoConfig as any).preinstallTools, repoConfig.useDockerCompose);
  if (tools.length === 0 && repoConfig.aptPackages.length === 0) {
    await appendLog(previewId, "[PP] No preinstall options selected.\n");
    return;
  }

  await appendLog(previewId, `[PP] Installing selected preinstall options: ${tools.length ? tools.join(", ") : "custom apt packages only"}.\n`);

  const apt = `sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300`;
  const aptPackages = new Set<string>();
  if (tools.includes("python")) ["python3", "python3-pip", "python3-venv"].forEach(p => aptPackages.add(p));
  if (tools.includes("go")) aptPackages.add("golang-go");
  if (tools.includes("lua")) ["lua5.4", "luarocks"].forEach(p => aptPackages.add(p));
  if (tools.includes("build-essential")) ["build-essential", "pkg-config"].forEach(p => aptPackages.add(p));
  for (const pkg of normalizeAptPackages(repoConfig.aptPackages)) aptPackages.add(pkg);

  if (aptPackages.size > 0) {
    await runSshStep(previewId, sshSession, `${APT_WAIT}; ${apt} update && ${apt} install -y ${[...aptPackages].join(" ")}`);
  }

  if (tools.includes("docker")) {
    await runSshStep(
      previewId,
      sshSession,
      `${APT_WAIT}; if ! command -v docker >/dev/null 2>&1; then curl -fsSL https://get.docker.com | sudo sh; fi; ` +
        `sudo systemctl enable docker && sudo systemctl start docker; ${APT_WAIT}; ${apt} update && ${apt} install -y docker-compose-plugin; sudo usermod -aG docker ubuntu`,
    );
  }

  if (tools.includes("node")) {
    const nodeVersion = String((repoConfig as any).nodeVersion || "lts/*");
    await runSshStep(
      previewId,
      sshSession,
      bashLc(
        `if [ ! -s "$HOME/.nvm/nvm.sh" ]; then curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash; fi; ` +
          `export NVM_DIR="$HOME/.nvm"; source "$NVM_DIR/nvm.sh"; ` +
          `if [ -f /opt/app/.nvmrc ]; then cd /opt/app && nvm install && nvm use; ` +
          `else nvm install ${shellQuote(nodeVersion)} && nvm alias default ${shellQuote(nodeVersion)} && nvm use default; fi`,
      ),
    );
  }
}

// Wait for the freshly-launched app to come up. `nohup ... &` detaches the
// process and routes its output to pp.log, so nothing from the server reaches
// the preview logs — a clean start and an instant crash look identical. Poll
// until either the process dies or *anything* accepts a TCP connection on the
// app port (a bare connect, not an HTTP 2xx — the app owns the port either way),
// then surface the first lines of pp.log so the startup is visible. Polls from
// Node in short bursts because a single SSH exec is capped at 30s (see ssh.ts),
// so a long in-shell loop would trip that timeout.
const HEALTH_TIMEOUT_MS = 60_000;
const HEALTH_INTERVAL_MS = 2_000;

async function healthCheck(previewId: number, sshSession: SshSession, pid: number, port: number) {
  await appendLog(previewId, `[PP] Waiting up to ${HEALTH_TIMEOUT_MS / 1000}s for the app to respond on port ${port}...\n`);

  const start = Date.now();
  let healthy = false;
  let exited = false;

  while (Date.now() - start < HEALTH_TIMEOUT_MS) {
    checkAbortSession(sshSession);
    // exit 2 = process gone, 0 = port accepted a connection, 1 = not yet.
    // `/dev/tcp` is a bash builtin, so no nc/curl dependency; the connect runs
    // in a subshell so its fd can't leak into later commands on this channel.
    const probe = await sshSession.exec(
      `bash -c 'kill -0 ${pid} 2>/dev/null || exit 2; (exec 3<>/dev/tcp/127.0.0.1/${port}) 2>/dev/null && exit 0 || exit 1'`,
    );
    if (probe.code === 0) { healthy = true; break; }
    if (probe.code === 2) { exited = true; break; }
    await sleep(HEALTH_INTERVAL_MS);
  }

  const tail = await sshSession.exec(`tail -n 40 /opt/app/pp.log 2>/dev/null`);
  await appendLog(previewId, `--- server startup output (pp.log) ---\n`);
  if (tail.stdout) await appendLog(previewId, tail.stdout);
  await appendLog(previewId, `\n---\n`);

  if (healthy) {
    await appendLog(previewId, `[PP] Health check passed — port ${port} is accepting connections (pid ${pid}).\n`);
  } else if (exited) {
    throw new Error(`Server process ${pid} exited during startup — see pp.log output above.`);
  } else {
    throw new Error(`Health check timed out — nothing responded on port ${port} within ${HEALTH_TIMEOUT_MS / 1000}s. See pp.log output above.`);
  }
}

function sleep(ms: number) {
  return new Promise(r => setTimeout(r, ms));
}

async function runSshStep(previewId: number, sshSession: SshSession, command: string, throwOnFail = true) {
  checkAbortSession(sshSession);
  await appendLog(previewId, `$ ${command}\n`);
  const res = await sshSession.exec(command);
  if (res.stdout) await appendLog(previewId, res.stdout);
  if (res.stderr) await appendLog(previewId, res.stderr);
  if (throwOnFail && res.code !== 0) {
    throw new Error(`Command failed with exit code ${res.code}: ${command}`);
  }
  return res;
}

// Like runSshStep, but for git commands that must carry the Gitea auth header.
// The real command embeds the base64 credential inline (per-command, never
// persisted to .git/config); `displayCommand` is the credential-free form we
// echo to the logs, and any secret leaking into stdout/stderr is masked. This
// mirrors the masking the initial clone already does.
async function runGitStep(
  previewId: number,
  sshSession: SshSession,
  realCommand: string,
  displayCommand: string,
  ...maskValues: string[]
) {
  checkAbortSession(sshSession);
  await appendLog(previewId, `$ ${displayCommand}\n`);
  const res = await sshSession.exec(realCommand);
  const mask = (s: string) => maskValues.reduce((acc, m) => (m ? acc.split(m).join("****") : acc), s);
  if (res.stdout) await appendLog(previewId, mask(res.stdout));
  if (res.stderr) await appendLog(previewId, mask(res.stderr));
  if (res.code !== 0) {
    throw new Error(`Command failed with exit code ${res.code}: ${displayCommand}`);
  }
  return res;
}

// Builds the `-c http.extraHeader=...` git option that authenticates a single
// git invocation against a private Gitea repo. Empty when there is no PAT
// (public repos clone/fetch anonymously).
function giteaAuthOption(authHeader: string, giteaPat: string): string {
  return giteaPat ? `-c http.extraHeader='Authorization: Basic ${authHeader}' ` : "";
}

function checkAbortSession(session: SshSession) {
  if (session.aborted) throw new Error("ABORTED");
}

const abortSignals = new Set<number>();

export function signalAbort(previewId: number) {
  abortSignals.add(previewId);
  abortJobForPreview(previewId);
}

function checkAbort(previewId: number) {
  if (abortSignals.has(previewId)) {
    abortSignals.delete(previewId);
    throw new Error("ABORTED");
  }
}

export async function stopPreview(previewId: number, status: "STOPPED" | "FAILED" = "STOPPED", reason = "Stopped", waitForCleanup = false) {
  const preview = await prisma.preview.findUnique({
    where: { id: previewId },
    include: { repoConfig: { include: { user: true } } },
  });
  if (!preview) return;

  const user = (preview.repoConfig as any).user as User;
  const repoConfig = preview.repoConfig;

  if (preview.instanceId) {
    const ec2 = makeEc2Client(user);
    const instanceId = preview.instanceId;
    // Terminate instance first, then clean up key pair and security group
    try {
      await terminateInstance(ec2, instanceId);
    } catch (e) {
      log.warn({ e, instanceId }, "Failed to terminate instance");
    }
    // Delete key pair immediately (doesn't depend on instance state)
    try {
      await deleteKeyPairAws(ec2, `pp-preview-${previewId}`);
    } catch {}
    // The security group can't be deleted until the instance's ENI is released,
    // which only happens once the instance is fully terminated. Wait for that,
    // then delete (deleteSecurityGroupAws also retries on DependencyViolation).
    const cleanupSecurityGroup = async () => {
      try {
        await waitForInstanceTerminated(ec2, instanceId);
        await deleteSecurityGroupAws(ec2, `pp-preview-${previewId}`);
      } catch (e) {
        log.warn({ e, previewId }, "Failed to delete security group after termination");
      }
    };
    if (waitForCleanup) await cleanupSecurityGroup();
    else void cleanupSecurityGroup();
  }

  // Close out the live cost session: fold the running instance's accrued cost
  // into the finalized total and clear the launch timestamp so it stops billing.
  const stoppedAt = new Date();
  const finalCostUsd = computePreviewCostUsd(
    {
      accumulatedCostUsd: preview.accumulatedCostUsd,
      instanceLaunchedAt: preview.instanceLaunchedAt,
      instanceType: preview.instanceType,
    },
    stoppedAt,
  );

  const stopReason = reason.trim().slice(0, 128) || "Stopped";

  await prisma.preview.update({
    where: { id: previewId },
    data: {
      status,
      stopReason,
      stoppedAt,
      sshPrivateKey: null,
      sshKeyName: null,
      instanceId: null,
      instanceLaunchedAt: null,
      accumulatedCostUsd: finalCostUsd,
    },
  });

  const stoppedBody = buildPrCommentBody({
    owner: repoConfig.repoOwner,
    repo: repoConfig.repoName,
    prNumber: preview.prNumber,
    status: `⚫ Stopped (${stopReason})`,
    commitSha: preview.commitSha,
    updatedAt: new Date(),
    ppBaseUrl: env.PP_BASE_URL,
  });

  if (preview.giteaCommentId) {
    try {
      await updateComment(user, repoConfig.repoOwner, repoConfig.repoName, preview.giteaCommentId, stoppedBody);
    } catch {}
  }
}
