diff --git a/README.md b/README.md index 696074c..b33cddf 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Tiny API that clones a Git repo and counts its non-empty lines of code. - Returns both snapshots and their total/per-language LOC delta. - `GET /health` - Health plus queue/cache stats. +- `GET /metrics` + - Prometheus-compatible request, cache, scan, file, and line counters. Protected by the API key when one is configured. ## Auth @@ -70,6 +72,12 @@ curl "http://localhost:3000/ssh/public-key?ssh_key=loc_via_git_ed25519" - The service also sweeps stale temp directories in case a process dies mid-scan. - Cache entries live in memory only and expire after `CACHE_TTL_MINUTES`. +## Operations + +- Requests emit structured JSON logs with an opaque request ID, route, status, and duration. Query strings and API keys are never logged. +- The service handles `SIGINT` and `SIGTERM`: it stops accepting new connections, clears its cleanup timer, and exits after active connections drain (or 10 seconds). +- `/metrics` is compatible with Prometheus scraping. `/health` remains unauthenticated for container health checks. + ## Counting rules - Only text files are counted. diff --git a/src/app.ts b/src/app.ts index 98934e6..3020402 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,17 +4,20 @@ import rateLimit from "express-rate-limit"; import type { RuntimeConfig } from "./config.js"; import { HttpError } from "./errors.js"; import { asyncHandler, readOptionalString } from "./lib/http.js"; +import { Metrics } from "./services/metrics.js"; import type { ComparisonResult, CountRequest } from "./types.js"; type AppDependencies = { getHealth: () => Record; getPublicKey: (name: string | null) => Promise; getDefaultKeyName: () => string; + metrics?: Metrics; count: (request: CountRequest) => Promise; }; export function createApp(config: RuntimeConfig, deps: AppDependencies) { const app = express(); + const metrics = deps.metrics ?? new Metrics(); app.set("trust proxy", config.trustProxy); app.use(rateLimit({ @@ -24,6 +27,19 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) { legacyHeaders: false })); + app.use((req, res, next) => { + const startedAt = performance.now(); + const requestId = crypto.randomUUID(); + res.setHeader("x-request-id", requestId); + res.on("finish", () => { + const durationMs = Math.round(performance.now() - startedAt); + const route = getRouteLabel(req.path); + metrics.recordRequest(route, res.statusCode, durationMs); + console.log(JSON.stringify({ event: "request", requestId, route, status: res.statusCode, durationMs })); + }); + next(); + }); + app.get("/health", (_req, res) => { res.json({ ok: true, ...deps.getHealth() }); }); @@ -52,13 +68,17 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) { res.status(401).json({ error: "Unauthorized" }); }); + app.get("/metrics", (_req, res) => { + res.type("text/plain; version=0.0.4; charset=utf-8").send(metrics.renderPrometheus()); + }); + app.get("/loc.txt", asyncHandler(async (req, res) => { - const result = await deps.count(readCountRequest(req)); + const result = await countAndRecord(deps.count, metrics, readCountRequest(req)); res.type("text/plain").send(String(result.lineCount)); })); app.get("/loc", asyncHandler(async (req, res) => { - const result = await deps.count(readCountRequest(req)); + const result = await countAndRecord(deps.count, metrics, readCountRequest(req)); res.json(result); })); @@ -67,8 +87,8 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) { const head = readRequiredString(req.query.head, "head"); const shared = readCountRequest(req); const [baseResult, headResult] = await Promise.all([ - deps.count({ ...shared, ref: base }), - deps.count({ ...shared, ref: head }) + countAndRecord(deps.count, metrics, { ...shared, ref: base }), + countAndRecord(deps.count, metrics, { ...shared, ref: head }) ]); res.json(createComparison(baseResult, headResult)); })); @@ -76,12 +96,27 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) { app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => { const message = error instanceof Error ? error.message : "Unknown error"; const status = error instanceof HttpError ? error.statusCode : message.startsWith("Missing ") ? 400 : 500; + console.error(JSON.stringify({ event: "request_error", requestId: res.getHeader("x-request-id"), status, error: message })); res.status(status).json({ error: message }); }); return app; } +async function countAndRecord( + count: (request: CountRequest) => Promise, + metrics: Metrics, + request: CountRequest +): Promise { + const result = await count(request); + metrics.recordScan(result); + return result; +} + +function getRouteLabel(path: string): string { + return ["/health", "/loc", "/loc.txt", "/loc/diff", "/metrics", "/ssh/public-key"].includes(path) ? path : "other"; +} + function createComparison(base: import("./types.js").CountResult, head: import("./types.js").CountResult): ComparisonResult { const languages = new Map(base.languages.map((language) => [language.language, { ...language, files: -language.files, lines: -language.lines }])); for (const language of head.languages) { diff --git a/src/server.ts b/src/server.ts index d97b04a..5c22d95 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,11 +1,13 @@ import { loadConfig } from "./config.js"; import { createApp } from "./app.js"; import { KeyManager } from "./services/key-manager.js"; +import { Metrics } from "./services/metrics.js"; import { RepoCounterService } from "./services/repo-counter.js"; const config = loadConfig(); const keyManager = new KeyManager(config); const repoCounter = new RepoCounterService(config, keyManager); +const metrics = new Metrics(); await keyManager.initialize(); await repoCounter.initialize(); @@ -14,9 +16,29 @@ const app = createApp(config, { getHealth: () => repoCounter.getHealth(), getPublicKey: (name) => keyManager.getPublicKey(name), getDefaultKeyName: () => keyManager.getDefaultKeyName(), + metrics, count: (request) => repoCounter.count(request) }); -app.listen(config.port, () => { - console.log(`loc-via-git listening on ${config.port}`); +const server = app.listen(config.port, () => { + console.log(JSON.stringify({ event: "server_started", port: config.port })); }); + +let shuttingDown = false; +for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.on(signal, () => { + if (shuttingDown) { + return; + } + shuttingDown = true; + console.log(JSON.stringify({ event: "shutdown_started", signal })); + repoCounter.close(); + const forceExit = setTimeout(() => process.exit(1), 10_000); + forceExit.unref(); + server.close(() => { + clearTimeout(forceExit); + console.log(JSON.stringify({ event: "shutdown_complete", signal })); + process.exit(0); + }); + }); +} diff --git a/src/services/metrics.ts b/src/services/metrics.ts new file mode 100644 index 0000000..78eed1c --- /dev/null +++ b/src/services/metrics.ts @@ -0,0 +1,57 @@ +import type { CountResult } from "../types.js"; + +export class Metrics { + private readonly requestCounts = new Map(); + private requestDurationMs = 0; + private scanCount = 0; + private scanDurationMs = 0; + private cachedScanCount = 0; + private countedFiles = 0; + private countedLines = 0; + + recordRequest(route: string, status: number, durationMs: number): void { + const key = `${route}\u0000${status}`; + this.requestCounts.set(key, (this.requestCounts.get(key) ?? 0) + 1); + this.requestDurationMs += durationMs; + } + + recordScan(result: CountResult): void { + this.scanCount += 1; + this.scanDurationMs += result.durationMs; + this.cachedScanCount += Number(result.cached); + this.countedFiles += result.fileCount; + this.countedLines += result.lineCount; + } + + renderPrometheus(): string { + const lines = [ + "# HELP loc_via_git_http_requests_total HTTP responses by route and status.", + "# TYPE loc_via_git_http_requests_total counter" + ]; + for (const [key, count] of this.requestCounts) { + const [route, status] = key.split("\u0000"); + lines.push(`loc_via_git_http_requests_total{route="${route}",status="${status}"} ${count}`); + } + lines.push( + "# HELP loc_via_git_http_request_duration_ms_total Total HTTP request duration in milliseconds.", + "# TYPE loc_via_git_http_request_duration_ms_total counter", + `loc_via_git_http_request_duration_ms_total ${this.requestDurationMs}`, + "# HELP loc_via_git_scans_total Completed LOC scans.", + "# TYPE loc_via_git_scans_total counter", + `loc_via_git_scans_total ${this.scanCount}`, + "# HELP loc_via_git_cached_scans_total Completed scans served from cache.", + "# TYPE loc_via_git_cached_scans_total counter", + `loc_via_git_cached_scans_total ${this.cachedScanCount}`, + "# HELP loc_via_git_scan_duration_ms_total Total scan duration in milliseconds.", + "# TYPE loc_via_git_scan_duration_ms_total counter", + `loc_via_git_scan_duration_ms_total ${this.scanDurationMs}`, + "# HELP loc_via_git_counted_files_total Files included in completed scan results.", + "# TYPE loc_via_git_counted_files_total counter", + `loc_via_git_counted_files_total ${this.countedFiles}`, + "# HELP loc_via_git_counted_lines_total Non-empty lines included in completed scan results.", + "# TYPE loc_via_git_counted_lines_total counter", + `loc_via_git_counted_lines_total ${this.countedLines}` + ); + return `${lines.join("\n")}\n`; + } +} diff --git a/src/services/repo-counter.ts b/src/services/repo-counter.ts index 4e7e224..43c0c66 100644 --- a/src/services/repo-counter.ts +++ b/src/services/repo-counter.ts @@ -20,6 +20,7 @@ export class RepoCounterService { private readonly cache = new Map(); private readonly inFlight = new Map>>(); private readonly semaphore: Semaphore; + private cleanupTimer: NodeJS.Timeout | null = null; constructor( private readonly config: RuntimeConfig, @@ -44,10 +45,18 @@ export class RepoCounterService { await mkdir(this.config.tempRoot, { recursive: true }); await this.cleanupStaleTempDirs(); - setInterval(() => { + this.cleanupTimer = setInterval(() => { this.clearExpiredCache(); void this.cleanupStaleTempDirs(); - }, this.config.cacheSweepIntervalMs).unref(); + }, this.config.cacheSweepIntervalMs); + this.cleanupTimer.unref(); + } + + close(): void { + if (this.cleanupTimer) { + clearInterval(this.cleanupTimer); + this.cleanupTimer = null; + } } async count(request: CountRequest): Promise { diff --git a/tests/app.test.js b/tests/app.test.js index dff5596..3cd455d 100644 --- a/tests/app.test.js +++ b/tests/app.test.js @@ -106,6 +106,25 @@ test("GET /loc accepts api_key query param as an unsafe fallback", async () => { assert.equal(response.body.lineCount, 5); }); +test("GET /metrics is authenticated and exposes Prometheus counters", async () => { + const app = createApp({ ...baseConfig, apiKey: "secret" }, { + getHealth: () => ({ cacheEntries: 0, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }), + getPublicKey: async () => "ssh-ed25519 AAAA", + getDefaultKeyName: () => "loc_via_git_ed25519", + count: async () => { + throw new Error("not used"); + } + }); + + const unauthenticated = await request(app).get("/metrics"); + const authenticated = await request(app).get("/metrics").set("x-api-key", "secret"); + + assert.equal(unauthenticated.status, 401); + assert.equal(authenticated.status, 200); + assert.match(authenticated.text, /loc_via_git_http_requests_total/); + assert.match(authenticated.text, /loc_via_git_scans_total 0/); +}); + test("GET /loc/diff returns aggregate and language deltas", async () => { const app = createApp(baseConfig, { getHealth: () => ({ cacheEntries: 0, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }), diff --git a/tests/repo-counter.test.js b/tests/repo-counter.test.js new file mode 100644 index 0000000..ae6c313 --- /dev/null +++ b/tests/repo-counter.test.js @@ -0,0 +1,148 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { KeyManager } from "../dist/services/key-manager.js"; +import { RepoCounterService } from "../dist/services/repo-counter.js"; + +async function createHarness(options = {}) { + const root = await mkdtemp(path.join(os.tmpdir(), "loc-via-git-test-")); + const fixture = path.join(root, "fixture"); + const bin = path.join(root, "bin"); + const cloneLog = path.join(root, "clone-log"); + await mkdir(fixture, { recursive: true }); + await mkdir(bin); + await writeFile(path.join(bin, "git"), `#!/usr/bin/env node +import { appendFileSync, cpSync } from "node:fs"; +const args = process.argv.slice(2); +const command = args.includes("clone") ? "clone" : args.includes("rev-parse") ? "rev-parse" : "other"; +appendFileSync(process.env.LOC_TEST_CLONE_LOG, args.join(" ") + "\\n"); +if (command === "clone") { + if (process.env.LOC_TEST_HANG === "1") setInterval(() => {}, 1000); + if (process.env.LOC_TEST_FAIL === "1") process.exit(2); + cpSync(process.env.LOC_TEST_FIXTURE, args.at(-1), { recursive: true }); +} else if (command === "rev-parse") { + process.stdout.write("fixture-commit\\n"); +} +`, { mode: 0o755 }); + + const config = { + allowedGitHosts: ["example.test"], + apiKey: "", + cacheSweepIntervalMs: 60_000, + cacheTtlMs: 60_000, + cloneTimeoutMs: 100, + defaultSshKeyName: "", + generateSshKeyIfMissing: false, + maxConcurrentScans: 2, + maxFilesPerScan: 100, + maxFileSizeBytes: 1024 * 1024, + maxScanBytes: 1024 * 1024, + port: 3000, + rateLimitMax: 30, + rateLimitWindowMs: 60_000, + sshKeysDir: path.join(root, "keys"), + tempRoot: path.join(root, "tmp"), + trustProxy: false, + ...options + }; + const originalPath = process.env.PATH; + process.env.PATH = `${bin}:${originalPath}`; + process.env.LOC_TEST_FIXTURE = fixture; + process.env.LOC_TEST_CLONE_LOG = cloneLog; + const keyManager = new KeyManager(config); + const service = new RepoCounterService(config, keyManager); + await keyManager.initialize(); + await service.initialize(); + + return { + cloneLog, + config, + fixture, + root, + service, + async cleanup() { + process.env.PATH = originalPath; + delete process.env.LOC_TEST_FIXTURE; + delete process.env.LOC_TEST_CLONE_LOG; + delete process.env.LOC_TEST_HANG; + delete process.env.LOC_TEST_FAIL; + await rm(root, { recursive: true, force: true }); + } + }; +} + +test("scans fixture repositories, caches duplicate work, and cleans clone directories", async () => { + const harness = await createHarness(); + try { + await writeFile(path.join(harness.fixture, "main.ts"), "const value = 1;\n\nexport { value };\n"); + await mkdir(path.join(harness.fixture, "dist")); + await writeFile(path.join(harness.fixture, "dist", "bundle.min.js"), "ignored\n"); + + const request = { repo: "https://example.test/demo.git", ref: null, sshKey: null }; + const [first, second] = await Promise.all([harness.service.count(request), harness.service.count(request)]); + const third = await harness.service.count(request); + + assert.equal(first.commit, "fixture-commit"); + assert.equal(second.lineCount, 2); + assert.equal(third.cached, true); + const commands = await readFile(harness.cloneLog, "utf8"); + assert.equal(commands.split("\n").filter((line) => line.includes(" clone ")).length, 1); + assert.deepEqual(await readdir(harness.config.tempRoot), []); + } finally { + await harness.cleanup(); + } +}); + +test("checks out requested refs and cleans up failed clones", async () => { + const harness = await createHarness(); + try { + await writeFile(path.join(harness.fixture, "main.ts"), "export {};\n"); + await harness.service.count({ repo: "https://example.test/demo.git", ref: "release-1", sshKey: null }); + const commands = await readFile(harness.cloneLog, "utf8"); + assert.match(commands, /fetch --depth 1 origin release-1/); + assert.match(commands, /checkout FETCH_HEAD/); + + process.env.LOC_TEST_FAIL = "1"; + await assert.rejects( + harness.service.count({ repo: "https://example.test/failure.git", ref: null, sshKey: null }), + { statusCode: 400 } + ); + assert.deepEqual(await readdir(harness.config.tempRoot), []); + } finally { + await harness.cleanup(); + } +}); + +test("rejects oversized files and still removes the temporary clone", async () => { + const harness = await createHarness({ maxFileSizeBytes: 10 }); + try { + await writeFile(path.join(harness.fixture, "large.ts"), "export const value = 123;\n"); + await assert.rejects( + harness.service.count({ repo: "https://example.test/demo.git", ref: null, sshKey: null }), + { statusCode: 413 } + ); + assert.deepEqual(await readdir(harness.config.tempRoot), []); + } finally { + await harness.cleanup(); + } +}); + +test("times out stalled clones and rejects unsupported repository URLs", async () => { + const harness = await createHarness({ cloneTimeoutMs: 50 }); + try { + process.env.LOC_TEST_HANG = "1"; + await assert.rejects( + harness.service.count({ repo: "https://example.test/demo.git", ref: null, sshKey: null }), + { statusCode: 504 } + ); + await assert.rejects( + harness.service.count({ repo: "file:///private/repo", ref: null, sshKey: null }), + { statusCode: 400 } + ); + } finally { + await harness.cleanup(); + } +}); diff --git a/tests/server-lifecycle.test.js b/tests/server-lifecycle.test.js new file mode 100644 index 0000000..d7ad32d --- /dev/null +++ b/tests/server-lifecycle.test.js @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import net from "node:net"; +import test from "node:test"; + +async function getFreePort() { + const server = net.createServer(); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Could not allocate test port"); + } + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + return address.port; +} + +test("server shuts down cleanly after SIGTERM", async () => { + const port = await getFreePort(); + const child = spawn(process.execPath, ["dist/server.js"], { + cwd: process.cwd(), + env: { ...process.env, PORT: String(port), SSH_KEYS_DIR: `/tmp/loc-via-git-keys-${port}`, TMP_DIR: `/tmp/loc-via-git-${port}` }, + stdio: ["ignore", "pipe", "pipe"] + }); + let output = ""; + child.stdout.on("data", (chunk) => { + output += chunk.toString(); + }); + + try { + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error(`Server did not start: ${output}`)), 2_000); + const interval = setInterval(() => { + if (output.includes("server_started")) { + clearTimeout(timeout); + clearInterval(interval); + resolve(); + } + }, 10); + }); + child.kill("SIGTERM"); + const code = await new Promise((resolve) => child.once("exit", resolve)); + assert.equal(code, 0); + assert.match(output, /shutdown_complete/); + } finally { + child.kill("SIGKILL"); + } +});