Add observability and integration coverage (#2)

Co-authored-by: luna <clawy@reversed.dev>
Co-committed-by: luna <clawy@reversed.dev>

README.md

@@ -25,6 +25,8 @@ Tiny API that clones a Git repo and counts its non-empty lines of code.
   - Returns both snapshots and their total/per-language LOC delta.
 - `GET /health`
   - Health plus queue/cache stats.
+- `GET /metrics`
+  - Prometheus-compatible request, cache, scan, file, and line counters. Protected by the API key when one is configured.
 
 ## Auth
 
@@ -70,6 +72,12 @@ curl "http://localhost:3000/ssh/public-key?ssh_key=loc_via_git_ed25519"
 - The service also sweeps stale temp directories in case a process dies mid-scan.
 - Cache entries live in memory only and expire after `CACHE_TTL_MINUTES`.
 
+## Operations
+
+- Requests emit structured JSON logs with an opaque request ID, route, status, and duration. Query strings and API keys are never logged.
+- The service handles `SIGINT` and `SIGTERM`: it stops accepting new connections, clears its cleanup timer, and exits after active connections drain (or 10 seconds).
+- `/metrics` is compatible with Prometheus scraping. `/health` remains unauthenticated for container health checks.
+
 ## Counting rules
 
 - Only text files are counted.

src/app.ts

@@ -4,17 +4,20 @@ import rateLimit from "express-rate-limit";
 import type { RuntimeConfig } from "./config.js";
 import { HttpError } from "./errors.js";
 import { asyncHandler, readOptionalString } from "./lib/http.js";
+import { Metrics } from "./services/metrics.js";
 import type { ComparisonResult, CountRequest } from "./types.js";
 
 type AppDependencies = {
   getHealth: () => Record<string, number>;
   getPublicKey: (name: string | null) => Promise<string>;
   getDefaultKeyName: () => string;
+  metrics?: Metrics;
   count: (request: CountRequest) => Promise<import("./types.js").CountResult>;
 };
 
 export function createApp(config: RuntimeConfig, deps: AppDependencies) {
   const app = express();
+  const metrics = deps.metrics ?? new Metrics();
   app.set("trust proxy", config.trustProxy);
 
   app.use(rateLimit({
@@ -24,6 +27,19 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
     legacyHeaders: false
   }));
 
+  app.use((req, res, next) => {
+    const startedAt = performance.now();
+    const requestId = crypto.randomUUID();
+    res.setHeader("x-request-id", requestId);
+    res.on("finish", () => {
+      const durationMs = Math.round(performance.now() - startedAt);
+      const route = getRouteLabel(req.path);
+      metrics.recordRequest(route, res.statusCode, durationMs);
+      console.log(JSON.stringify({ event: "request", requestId, route, status: res.statusCode, durationMs }));
+    });
+    next();
+  });
+
   app.get("/health", (_req, res) => {
     res.json({ ok: true, ...deps.getHealth() });
   });
@@ -52,13 +68,17 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
     res.status(401).json({ error: "Unauthorized" });
   });
 
+  app.get("/metrics", (_req, res) => {
+    res.type("text/plain; version=0.0.4; charset=utf-8").send(metrics.renderPrometheus());
+  });
+
   app.get("/loc.txt", asyncHandler(async (req, res) => {
-    const result = await deps.count(readCountRequest(req));
+    const result = await countAndRecord(deps.count, metrics, readCountRequest(req));
     res.type("text/plain").send(String(result.lineCount));
   }));
 
   app.get("/loc", asyncHandler(async (req, res) => {
-    const result = await deps.count(readCountRequest(req));
+    const result = await countAndRecord(deps.count, metrics, readCountRequest(req));
     res.json(result);
   }));
 
@@ -67,8 +87,8 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
     const head = readRequiredString(req.query.head, "head");
     const shared = readCountRequest(req);
     const [baseResult, headResult] = await Promise.all([
-      deps.count({ ...shared, ref: base }),
-      deps.count({ ...shared, ref: head })
+      countAndRecord(deps.count, metrics, { ...shared, ref: base }),
+      countAndRecord(deps.count, metrics, { ...shared, ref: head })
     ]);
     res.json(createComparison(baseResult, headResult));
   }));
@@ -76,12 +96,27 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
   app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => {
     const message = error instanceof Error ? error.message : "Unknown error";
     const status = error instanceof HttpError ? error.statusCode : message.startsWith("Missing ") ? 400 : 500;
+    console.error(JSON.stringify({ event: "request_error", requestId: res.getHeader("x-request-id"), status, error: message }));
     res.status(status).json({ error: message });
   });
 
   return app;
 }
 
+async function countAndRecord(
+  count: (request: CountRequest) => Promise<import("./types.js").CountResult>,
+  metrics: Metrics,
+  request: CountRequest
+): Promise<import("./types.js").CountResult> {
+  const result = await count(request);
+  metrics.recordScan(result);
+  return result;
+}
+
+function getRouteLabel(path: string): string {
+  return ["/health", "/loc", "/loc.txt", "/loc/diff", "/metrics", "/ssh/public-key"].includes(path) ? path : "other";
+}
+
 function createComparison(base: import("./types.js").CountResult, head: import("./types.js").CountResult): ComparisonResult {
   const languages = new Map(base.languages.map((language) => [language.language, { ...language, files: -language.files, lines: -language.lines }]));
   for (const language of head.languages) {

src/server.ts

@@ -1,11 +1,13 @@
 import { loadConfig } from "./config.js";
 import { createApp } from "./app.js";
 import { KeyManager } from "./services/key-manager.js";
+import { Metrics } from "./services/metrics.js";
 import { RepoCounterService } from "./services/repo-counter.js";
 
 const config = loadConfig();
 const keyManager = new KeyManager(config);
 const repoCounter = new RepoCounterService(config, keyManager);
+const metrics = new Metrics();
 
 await keyManager.initialize();
 await repoCounter.initialize();
@@ -14,9 +16,29 @@ const app = createApp(config, {
   getHealth: () => repoCounter.getHealth(),
   getPublicKey: (name) => keyManager.getPublicKey(name),
   getDefaultKeyName: () => keyManager.getDefaultKeyName(),
+  metrics,
   count: (request) => repoCounter.count(request)
 });
 
-app.listen(config.port, () => {
-  console.log(`loc-via-git listening on ${config.port}`);
+const server = app.listen(config.port, () => {
+  console.log(JSON.stringify({ event: "server_started", port: config.port }));
 });
+
+let shuttingDown = false;
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+  process.on(signal, () => {
+    if (shuttingDown) {
+      return;
+    }
+    shuttingDown = true;
+    console.log(JSON.stringify({ event: "shutdown_started", signal }));
+    repoCounter.close();
+    const forceExit = setTimeout(() => process.exit(1), 10_000);
+    forceExit.unref();
+    server.close(() => {
+      clearTimeout(forceExit);
+      console.log(JSON.stringify({ event: "shutdown_complete", signal }));
+      process.exit(0);
+    });
+  });
+}

src/services/metrics.ts

@@ -0,0 +1,57 @@
+import type { CountResult } from "../types.js";
+
+export class Metrics {
+  private readonly requestCounts = new Map<string, number>();
+  private requestDurationMs = 0;
+  private scanCount = 0;
+  private scanDurationMs = 0;
+  private cachedScanCount = 0;
+  private countedFiles = 0;
+  private countedLines = 0;
+
+  recordRequest(route: string, status: number, durationMs: number): void {
+    const key = `${route}\u0000${status}`;
+    this.requestCounts.set(key, (this.requestCounts.get(key) ?? 0) + 1);
+    this.requestDurationMs += durationMs;
+  }
+
+  recordScan(result: CountResult): void {
+    this.scanCount += 1;
+    this.scanDurationMs += result.durationMs;
+    this.cachedScanCount += Number(result.cached);
+    this.countedFiles += result.fileCount;
+    this.countedLines += result.lineCount;
+  }
+
+  renderPrometheus(): string {
+    const lines = [
+      "# HELP loc_via_git_http_requests_total HTTP responses by route and status.",
+      "# TYPE loc_via_git_http_requests_total counter"
+    ];
+    for (const [key, count] of this.requestCounts) {
+      const [route, status] = key.split("\u0000");
+      lines.push(`loc_via_git_http_requests_total{route="${route}",status="${status}"} ${count}`);
+    }
+    lines.push(
+      "# HELP loc_via_git_http_request_duration_ms_total Total HTTP request duration in milliseconds.",
+      "# TYPE loc_via_git_http_request_duration_ms_total counter",
+      `loc_via_git_http_request_duration_ms_total ${this.requestDurationMs}`,
+      "# HELP loc_via_git_scans_total Completed LOC scans.",
+      "# TYPE loc_via_git_scans_total counter",
+      `loc_via_git_scans_total ${this.scanCount}`,
+      "# HELP loc_via_git_cached_scans_total Completed scans served from cache.",
+      "# TYPE loc_via_git_cached_scans_total counter",
+      `loc_via_git_cached_scans_total ${this.cachedScanCount}`,
+      "# HELP loc_via_git_scan_duration_ms_total Total scan duration in milliseconds.",
+      "# TYPE loc_via_git_scan_duration_ms_total counter",
+      `loc_via_git_scan_duration_ms_total ${this.scanDurationMs}`,
+      "# HELP loc_via_git_counted_files_total Files included in completed scan results.",
+      "# TYPE loc_via_git_counted_files_total counter",
+      `loc_via_git_counted_files_total ${this.countedFiles}`,
+      "# HELP loc_via_git_counted_lines_total Non-empty lines included in completed scan results.",
+      "# TYPE loc_via_git_counted_lines_total counter",
+      `loc_via_git_counted_lines_total ${this.countedLines}`
+    );
+    return `${lines.join("\n")}\n`;
+  }
+}

src/services/repo-counter.ts

@@ -20,6 +20,7 @@ export class RepoCounterService {
   private readonly cache = new Map<string, CacheEntry>();
   private readonly inFlight = new Map<string, Promise<Omit<CountResult, "cached">>>();
   private readonly semaphore: Semaphore;
+  private cleanupTimer: NodeJS.Timeout | null = null;
 
   constructor(
     private readonly config: RuntimeConfig,
@@ -44,10 +45,18 @@ export class RepoCounterService {
     await mkdir(this.config.tempRoot, { recursive: true });
     await this.cleanupStaleTempDirs();
 
-    setInterval(() => {
+    this.cleanupTimer = setInterval(() => {
       this.clearExpiredCache();
       void this.cleanupStaleTempDirs();
-    }, this.config.cacheSweepIntervalMs).unref();
+    }, this.config.cacheSweepIntervalMs);
+    this.cleanupTimer.unref();
+  }
+
+  close(): void {
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer);
+      this.cleanupTimer = null;
+    }
   }
 
   async count(request: CountRequest): Promise<CountResult> {

tests/app.test.js

@@ -106,6 +106,25 @@ test("GET /loc accepts api_key query param as an unsafe fallback", async () => {
   assert.equal(response.body.lineCount, 5);
 });
 
+test("GET /metrics is authenticated and exposes Prometheus counters", async () => {
+  const app = createApp({ ...baseConfig, apiKey: "secret" }, {
+    getHealth: () => ({ cacheEntries: 0, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }),
+    getPublicKey: async () => "ssh-ed25519 AAAA",
+    getDefaultKeyName: () => "loc_via_git_ed25519",
+    count: async () => {
+      throw new Error("not used");
+    }
+  });
+
+  const unauthenticated = await request(app).get("/metrics");
+  const authenticated = await request(app).get("/metrics").set("x-api-key", "secret");
+
+  assert.equal(unauthenticated.status, 401);
+  assert.equal(authenticated.status, 200);
+  assert.match(authenticated.text, /loc_via_git_http_requests_total/);
+  assert.match(authenticated.text, /loc_via_git_scans_total 0/);
+});
+
 test("GET /loc/diff returns aggregate and language deltas", async () => {
   const app = createApp(baseConfig, {
     getHealth: () => ({ cacheEntries: 0, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }),

tests/repo-counter.test.js

@@ -0,0 +1,148 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { KeyManager } from "../dist/services/key-manager.js";
+import { RepoCounterService } from "../dist/services/repo-counter.js";
+
+async function createHarness(options = {}) {
+  const root = await mkdtemp(path.join(os.tmpdir(), "loc-via-git-test-"));
+  const fixture = path.join(root, "fixture");
+  const bin = path.join(root, "bin");
+  const cloneLog = path.join(root, "clone-log");
+  await mkdir(fixture, { recursive: true });
+  await mkdir(bin);
+  await writeFile(path.join(bin, "git"), `#!/usr/bin/env node
+import { appendFileSync, cpSync } from "node:fs";
+const args = process.argv.slice(2);
+const command = args.includes("clone") ? "clone" : args.includes("rev-parse") ? "rev-parse" : "other";
+appendFileSync(process.env.LOC_TEST_CLONE_LOG, args.join(" ") + "\\n");
+if (command === "clone") {
+  if (process.env.LOC_TEST_HANG === "1") setInterval(() => {}, 1000);
+  if (process.env.LOC_TEST_FAIL === "1") process.exit(2);
+  cpSync(process.env.LOC_TEST_FIXTURE, args.at(-1), { recursive: true });
+} else if (command === "rev-parse") {
+  process.stdout.write("fixture-commit\\n");
+}
+`, { mode: 0o755 });
+
+  const config = {
+    allowedGitHosts: ["example.test"],
+    apiKey: "",
+    cacheSweepIntervalMs: 60_000,
+    cacheTtlMs: 60_000,
+    cloneTimeoutMs: 100,
+    defaultSshKeyName: "",
+    generateSshKeyIfMissing: false,
+    maxConcurrentScans: 2,
+    maxFilesPerScan: 100,
+    maxFileSizeBytes: 1024 * 1024,
+    maxScanBytes: 1024 * 1024,
+    port: 3000,
+    rateLimitMax: 30,
+    rateLimitWindowMs: 60_000,
+    sshKeysDir: path.join(root, "keys"),
+    tempRoot: path.join(root, "tmp"),
+    trustProxy: false,
+    ...options
+  };
+  const originalPath = process.env.PATH;
+  process.env.PATH = `${bin}:${originalPath}`;
+  process.env.LOC_TEST_FIXTURE = fixture;
+  process.env.LOC_TEST_CLONE_LOG = cloneLog;
+  const keyManager = new KeyManager(config);
+  const service = new RepoCounterService(config, keyManager);
+  await keyManager.initialize();
+  await service.initialize();
+
+  return {
+    cloneLog,
+    config,
+    fixture,
+    root,
+    service,
+    async cleanup() {
+      process.env.PATH = originalPath;
+      delete process.env.LOC_TEST_FIXTURE;
+      delete process.env.LOC_TEST_CLONE_LOG;
+      delete process.env.LOC_TEST_HANG;
+      delete process.env.LOC_TEST_FAIL;
+      await rm(root, { recursive: true, force: true });
+    }
+  };
+}
+
+test("scans fixture repositories, caches duplicate work, and cleans clone directories", async () => {
+  const harness = await createHarness();
+  try {
+    await writeFile(path.join(harness.fixture, "main.ts"), "const value = 1;\n\nexport { value };\n");
+    await mkdir(path.join(harness.fixture, "dist"));
+    await writeFile(path.join(harness.fixture, "dist", "bundle.min.js"), "ignored\n");
+
+    const request = { repo: "https://example.test/demo.git", ref: null, sshKey: null };
+    const [first, second] = await Promise.all([harness.service.count(request), harness.service.count(request)]);
+    const third = await harness.service.count(request);
+
+    assert.equal(first.commit, "fixture-commit");
+    assert.equal(second.lineCount, 2);
+    assert.equal(third.cached, true);
+    const commands = await readFile(harness.cloneLog, "utf8");
+    assert.equal(commands.split("\n").filter((line) => line.includes(" clone ")).length, 1);
+    assert.deepEqual(await readdir(harness.config.tempRoot), []);
+  } finally {
+    await harness.cleanup();
+  }
+});
+
+test("checks out requested refs and cleans up failed clones", async () => {
+  const harness = await createHarness();
+  try {
+    await writeFile(path.join(harness.fixture, "main.ts"), "export {};\n");
+    await harness.service.count({ repo: "https://example.test/demo.git", ref: "release-1", sshKey: null });
+    const commands = await readFile(harness.cloneLog, "utf8");
+    assert.match(commands, /fetch --depth 1 origin release-1/);
+    assert.match(commands, /checkout FETCH_HEAD/);
+
+    process.env.LOC_TEST_FAIL = "1";
+    await assert.rejects(
+      harness.service.count({ repo: "https://example.test/failure.git", ref: null, sshKey: null }),
+      { statusCode: 400 }
+    );
+    assert.deepEqual(await readdir(harness.config.tempRoot), []);
+  } finally {
+    await harness.cleanup();
+  }
+});
+
+test("rejects oversized files and still removes the temporary clone", async () => {
+  const harness = await createHarness({ maxFileSizeBytes: 10 });
+  try {
+    await writeFile(path.join(harness.fixture, "large.ts"), "export const value = 123;\n");
+    await assert.rejects(
+      harness.service.count({ repo: "https://example.test/demo.git", ref: null, sshKey: null }),
+      { statusCode: 413 }
+    );
+    assert.deepEqual(await readdir(harness.config.tempRoot), []);
+  } finally {
+    await harness.cleanup();
+  }
+});
+
+test("times out stalled clones and rejects unsupported repository URLs", async () => {
+  const harness = await createHarness({ cloneTimeoutMs: 50 });
+  try {
+    process.env.LOC_TEST_HANG = "1";
+    await assert.rejects(
+      harness.service.count({ repo: "https://example.test/demo.git", ref: null, sshKey: null }),
+      { statusCode: 504 }
+    );
+    await assert.rejects(
+      harness.service.count({ repo: "file:///private/repo", ref: null, sshKey: null }),
+      { statusCode: 400 }
+    );
+  } finally {
+    await harness.cleanup();
+  }
+});

tests/server-lifecycle.test.js

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import net from "node:net";
+import test from "node:test";
+
+async function getFreePort() {
+  const server = net.createServer();
+  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("Could not allocate test port");
+  }
+  await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
+  return address.port;
+}
+
+test("server shuts down cleanly after SIGTERM", async () => {
+  const port = await getFreePort();
+  const child = spawn(process.execPath, ["dist/server.js"], {
+    cwd: process.cwd(),
+    env: { ...process.env, PORT: String(port), SSH_KEYS_DIR: `/tmp/loc-via-git-keys-${port}`, TMP_DIR: `/tmp/loc-via-git-${port}` },
+    stdio: ["ignore", "pipe", "pipe"]
+  });
+  let output = "";
+  child.stdout.on("data", (chunk) => {
+    output += chunk.toString();
+  });
+
+  try {
+    await new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => reject(new Error(`Server did not start: ${output}`)), 2_000);
+      const interval = setInterval(() => {
+        if (output.includes("server_started")) {
+          clearTimeout(timeout);
+          clearInterval(interval);
+          resolve();
+        }
+      }, 10);
+    });
+    child.kill("SIGTERM");
+    const code = await new Promise((resolve) => child.once("exit", resolve));
+    assert.equal(code, 0);
+    assert.match(output, /shutdown_complete/);
+  } finally {
+    child.kill("SIGKILL");
+  }
+});