Really huge mass update; Getting everything up-to-spec and implementing a wide range of features

backend/src/services/ssh.ts

@@ -5,6 +5,11 @@ const log = createLogger("SSH");
 
 export interface SshSession {
   exec(command: string): Promise<{ stdout: string; stderr: string; code: number }>;
+  // Run a long-lived command (e.g. `tail -f`) and receive its stdout/stderr as
+  // it arrives. Unlike exec(), this never buffers or times out — the channel
+  // stays open until the returned handle's close() is called (or the connection
+  // ends). Used to stream live app logs off a running instance.
+  execStream(command: string, onData: (chunk: string) => void): { close(): void };
   close(): void;
   aborted: boolean;
   abort(): void;
@@ -28,6 +33,26 @@ export async function connectSsh(host: string, privateKey: string, maxWaitMs = 3
           if (aborted) throw new Error("SSH session aborted");
           return execOnConn(conn, command);
         },
+        execStream(command: string, onData: (chunk: string) => void) {
+          let stream: any = null;
+          let closed = false;
+          conn.exec(command, (err, s) => {
+            if (err) {
+              if (!closed) onData(`[app-logs] stream error: ${err.message}\n`);
+              return;
+            }
+            if (closed) { try { s.close(); } catch {} return; }
+            stream = s;
+            s.on("data", (d: Buffer) => onData(d.toString()));
+            s.stderr.on("data", (d: Buffer) => onData(d.toString()));
+          });
+          return {
+            close() {
+              closed = true;
+              try { stream?.close(); } catch {}
+            },
+          };
+        },
         close() {
           try { conn.end(); } catch {}
         },
@@ -95,14 +120,37 @@ function execOnConn(conn: Client, command: string): Promise<{ stdout: string; st
   });
 }
 
+// The `ubuntu` user accepts SSH almost immediately at boot — long before the
+// cloud-init UserData bootstrap has finished disabling apt-daily and installing
+// the small base dependency set PP needs before SSH-driven setup begins.
+// Proceeding early can still cause apt-lock failures in the deploy steps. Block until the bootstrap writes its
+// done marker; bail out fast (with the tail of its log) if it writes the failure
+// marker instead. Uses short one-shot execs so a stale channel just retries.
 export async function waitForBootstrap(session: SshSession, maxWaitMs = 600_000): Promise<void> {
   const start = Date.now();
   while (Date.now() - start < maxWaitMs) {
+    if (session.aborted) throw new Error("SSH session aborted");
     try {
-      const res = await session.exec("test -f /var/lib/pp-bootstrap-done && echo done || echo waiting");
-      if (res.stdout.trim() === "done") return;
-    } catch {
-      // transient SSH exec error (e.g. timeout, channel reset) — retry
+      const res = await session.exec(
+        "if [ -f /var/lib/pp-bootstrap-failed ]; then echo failed; " +
+        "elif [ -f /var/lib/pp-bootstrap-done ]; then echo done; else echo waiting; fi"
+      );
+      const state = res.stdout.trim();
+      if (state === "done") return;
+      if (state === "failed") {
+        let tail = "";
+        try {
+          const logRes = await session.exec("tail -n 30 /var/log/pp-bootstrap.log 2>/dev/null || true");
+          tail = logRes.stdout.trim();
+        } catch {
+          // best effort — the failure marker alone is enough to abort
+        }
+        throw new Error(`EC2 bootstrap failed${tail ? `:\n${tail}` : ""}`);
+      }
+    } catch (e: any) {
+      // A bootstrap failure we detected above must propagate; only swallow
+      // transient SSH exec errors (timeout, channel reset) and retry.
+      if (e?.message?.startsWith("EC2 bootstrap failed")) throw e;
     }
     await sleep(10_000);
   }