Harden scans and add LOC comparison endpoint (#1)

Co-authored-by: luna <clawy@reversed.dev>
Co-committed-by: luna <clawy@reversed.dev>

src/app.ts

@@ -4,7 +4,7 @@ import rateLimit from "express-rate-limit";
 import type { RuntimeConfig } from "./config.js";
 import { HttpError } from "./errors.js";
 import { asyncHandler, readOptionalString } from "./lib/http.js";
-import type { CountRequest } from "./types.js";
+import type { ComparisonResult, CountRequest } from "./types.js";
 
 type AppDependencies = {
   getHealth: () => Record<string, number>;
@@ -62,6 +62,17 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
     res.json(result);
   }));
 
+  app.get("/loc/diff", asyncHandler(async (req, res) => {
+    const base = readRequiredString(req.query.base, "base");
+    const head = readRequiredString(req.query.head, "head");
+    const shared = readCountRequest(req);
+    const [baseResult, headResult] = await Promise.all([
+      deps.count({ ...shared, ref: base }),
+      deps.count({ ...shared, ref: head })
+    ]);
+    res.json(createComparison(baseResult, headResult));
+  }));
+
   app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => {
     const message = error instanceof Error ? error.message : "Unknown error";
     const status = error instanceof HttpError ? error.statusCode : message.startsWith("Missing ") ? 400 : 500;
@@ -71,6 +82,25 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
   return app;
 }
 
+function createComparison(base: import("./types.js").CountResult, head: import("./types.js").CountResult): ComparisonResult {
+  const languages = new Map(base.languages.map((language) => [language.language, { ...language, files: -language.files, lines: -language.lines }]));
+  for (const language of head.languages) {
+    const delta = languages.get(language.language) ?? { language: language.language, files: 0, lines: 0 };
+    delta.files += language.files;
+    delta.lines += language.lines;
+    languages.set(language.language, delta);
+  }
+  return {
+    base,
+    head,
+    delta: {
+      fileCount: head.fileCount - base.fileCount,
+      lineCount: head.lineCount - base.lineCount,
+      languages: Array.from(languages.values()).filter((language) => language.files !== 0 || language.lines !== 0).sort((a, b) => b.lines - a.lines || a.language.localeCompare(b.language))
+    }
+  };
+}
+
 function readCountRequest(req: Request): CountRequest {
   const repo = typeof req.query.repo === "string" ? req.query.repo.trim() : "";
   if (!repo) {
@@ -83,3 +113,11 @@ function readCountRequest(req: Request): CountRequest {
     sshKey: readOptionalString(req.query.ssh_key)
   };
 }
+
+function readRequiredString(value: unknown, name: string): string {
+  const result = readOptionalString(value);
+  if (!result) {
+    throw new HttpError(400, `Missing ${name} query parameter`);
+  }
+  return result;
+}

src/config.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import { tmpdir } from "node:os";
 
 export type RuntimeConfig = {
+  allowedGitHosts: string[];
   apiKey: string;
   cacheSweepIntervalMs: number;
   cacheTtlMs: number;
@@ -9,6 +10,9 @@ export type RuntimeConfig = {
   defaultSshKeyName: string;
   generateSshKeyIfMissing: boolean;
   maxConcurrentScans: number;
+  maxFilesPerScan: number;
+  maxFileSizeBytes: number;
+  maxScanBytes: number;
   port: number;
   rateLimitMax: number;
   rateLimitWindowMs: number;
@@ -19,6 +23,7 @@ export type RuntimeConfig = {
 
 export function loadConfig(): RuntimeConfig {
   return {
+    allowedGitHosts: readEnvList("ALLOWED_GIT_HOSTS"),
     apiKey: readEnvString("API_KEY", ""),
     cacheSweepIntervalMs: readEnvMinutes("CACHE_SWEEP_INTERVAL_MINUTES", 5),
     cacheTtlMs: readEnvMinutes("CACHE_TTL_MINUTES", 5),
@@ -26,6 +31,9 @@ export function loadConfig(): RuntimeConfig {
     defaultSshKeyName: readEnvString("DEFAULT_SSH_KEY_NAME", "loc_via_git_ed25519"),
     generateSshKeyIfMissing: readEnvBoolean("GENERATE_SSH_KEY_IF_MISSING", false),
     maxConcurrentScans: readEnvNumber("MAX_CONCURRENT_SCANS", 4),
+    maxFilesPerScan: readEnvNumber("MAX_FILES_PER_SCAN", 20_000),
+    maxFileSizeBytes: readEnvNumber("MAX_FILE_SIZE_MB", 5) * 1024 * 1024,
+    maxScanBytes: readEnvNumber("MAX_SCAN_SIZE_MB", 100) * 1024 * 1024,
     port: readEnvNumber("PORT", 3000),
     rateLimitMax: readEnvNumber("RATE_LIMIT_MAX", 30),
     rateLimitWindowMs: readEnvMinutes("RATE_LIMIT_WINDOW_MINUTES", 5),
@@ -35,6 +43,11 @@ export function loadConfig(): RuntimeConfig {
   };
 }
 
+function readEnvList(name: string): string[] {
+  const raw = process.env[name]?.trim() ?? "";
+  return raw.split(",").map((value) => value.trim().toLowerCase()).filter(Boolean);
+}
+
 function readEnvMinutes(name: string, fallback: number): number {
   return readEnvNumber(name, fallback) * 60_000;
 }

src/file-filter.ts

@@ -51,7 +51,18 @@ const ignoredFileNames = new Set([
   "yarn.lock"
 ]);
 
+const ignoredDirectoryNames = new Set([
+  ".angular", ".cache", ".next", ".nuxt", ".output", ".parcel-cache", ".svelte-kit",
+  ".terraform", ".venv", "bower_components", "build", "coverage", "dist", "node_modules",
+  "out", "pods", "target", "vendor"
+]);
+
 export function shouldIgnoreCountFile(filePath: string): boolean {
   const fileName = path.basename(filePath).toLowerCase();
-  return ignoredFileNames.has(fileName);
+  const directories = path.dirname(filePath).toLowerCase().split(path.sep);
+  return ignoredFileNames.has(fileName)
+    || directories.some((directory) => ignoredDirectoryNames.has(directory))
+    || /\.(generated|designer|g|pb)\.[^.]+$/.test(fileName)
+    || /\.min\.(css|js|mjs|cjs)$/.test(fileName)
+    || fileName.endsWith(".snap");
 }

src/language.ts

@@ -1,6 +1,44 @@
 import path from "node:path";
 
 const extensionToLanguage: Record<string, string> = {
+  ".astro": "Astro",
+  ".bash": "Shell",
+  ".bat": "Batchfile",
+  ".cjs": "JavaScript",
+  ".clj": "Clojure",
+  ".cmake": "CMake",
+  ".coffee": "CoffeeScript",
+  ".dart": "Dart",
+  ".dockerfile": "Dockerfile",
+  ".ex": "Elixir",
+  ".exs": "Elixir",
+  ".fish": "Fish",
+  ".fs": "F#",
+  ".fsx": "F#",
+  ".gd": "GDScript",
+  ".graphql": "GraphQL",
+  ".groovy": "Groovy",
+  ".hs": "Haskell",
+  ".ini": "INI",
+  ".ipynb": "Jupyter Notebook",
+  ".jsx": "JavaScript React",
+  ".less": "Less",
+  ".lock": "Lockfile",
+  ".m": "Objective-C",
+  ".nim": "Nim",
+  ".pl": "Perl",
+  ".proto": "Protocol Buffers",
+  ".ps1": "PowerShell",
+  ".r": "R",
+  ".rkt": "Racket",
+  ".scala": "Scala",
+  ".sol": "Solidity",
+  ".styl": "Stylus",
+  ".tf": "Terraform",
+  ".v": "Verilog",
+  ".vala": "Vala",
+  ".wasm": "WebAssembly",
+  ".zig": "Zig",
   ".c": "C",
   ".cc": "C++",
   ".cpp": "C++",
@@ -13,7 +51,6 @@ const extensionToLanguage: Record<string, string> = {
   ".java": "Java",
   ".js": "JavaScript",
   ".json": "JSON",
-  ".jsx": "JavaScript React",
   ".kt": "Kotlin",
   ".lua": "Lua",
   ".md": "Markdown",
@@ -46,6 +83,10 @@ export function detectLanguage(filePath: string): string {
     return "Dockerfile";
   }
 
+  if (["makefile", "gnumakefile"].includes(fileName)) {
+    return "Makefile";
+  }
+
   if (fileName.endsWith(".d.ts")) {
     return "TypeScript";
   }

src/services/repo-counter.ts

@@ -1,6 +1,6 @@
 import { spawn } from "node:child_process";
 import { createReadStream } from "node:fs";
-import { mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
+import { mkdir, mkdtemp, open, readdir, rm, stat } from "node:fs/promises";
 import path from "node:path";
 
 import type { RuntimeConfig } from "../config.js";
@@ -51,7 +51,8 @@ export class RepoCounterService {
   }
 
   async count(request: CountRequest): Promise<CountResult> {
-    const cacheKey = JSON.stringify(request);
+    const validatedRequest = this.validateRequest(request);
+    const cacheKey = JSON.stringify(validatedRequest);
     this.clearExpiredCache();
 
     const cached = this.cache.get(cacheKey);
@@ -66,7 +67,7 @@ export class RepoCounterService {
     }
 
     const task = this.semaphore.use(async () => {
-      const value = await this.cloneAndCount(request);
+      const value = await this.cloneAndCount(validatedRequest);
       this.cache.set(cacheKey, {
         value,
         expiresAt: Date.now() + this.config.cacheTtlMs
@@ -90,10 +91,14 @@ export class RepoCounterService {
 
     try {
       await this.runGitClone(request.repo, repoDir, request.ref, request.sshKey);
-      const stats = await countDirectory(repoDir);
+      const [stats, commit] = await Promise.all([
+        countDirectory(repoDir, this.config),
+        runCommand("git", ["-C", repoDir, "rev-parse", "HEAD"], { ...process.env }, this.config.cloneTimeoutMs, true)
+      ]);
 
       return {
-        repo: request.repo,
+        commit: commit.trim(),
+        repo: redactRepoUrl(request.repo),
         ref: request.ref,
         sshKey: request.sshKey,
         lineCount: stats.lineCount,
@@ -108,14 +113,16 @@ export class RepoCounterService {
   }
 
   private async runGitClone(repo: string, targetDir: string, ref: string | null, sshKey: string | null): Promise<void> {
-    const env = { ...process.env };
-    const keyPath = await this.keyManager.resolvePrivateKeyPath(sshKey);
+    const env: NodeJS.ProcessEnv = { ...process.env, GIT_TERMINAL_PROMPT: "0" };
+    const keyPath = sshKey || isSshRepository(repo)
+      ? await this.keyManager.resolvePrivateKeyPath(sshKey)
+      : null;
 
     if (keyPath) {
       env.GIT_SSH_COMMAND = `ssh -i "${keyPath}" -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new`;
     }
 
-    await runCommand("git", ["clone", "--depth", "1", "--single-branch", repo, targetDir], env, this.config.cloneTimeoutMs);
+    await runCommand("git", ["-c", "protocol.file.allow=never", "clone", "--depth", "1", "--single-branch", repo, targetDir], env, this.config.cloneTimeoutMs);
 
     if (ref) {
       await runCommand("git", ["-C", targetDir, "fetch", "--depth", "1", "origin", ref], env, this.config.cloneTimeoutMs);
@@ -123,6 +130,14 @@ export class RepoCounterService {
     }
   }
 
+  private validateRequest(request: CountRequest): CountRequest {
+    const host = getGitHost(request.repo);
+    if (this.config.allowedGitHosts.length > 0 && !this.config.allowedGitHosts.includes("*") && !this.config.allowedGitHosts.includes(host)) {
+      throw new HttpError(403, "Git host is not allowed");
+    }
+    return request;
+  }
+
   private async cleanupStaleTempDirs(): Promise<void> {
     const entries = await readdir(this.config.tempRoot, { withFileTypes: true }).catch(() => []);
     const staleBefore = Date.now() - this.config.cloneTimeoutMs * 2;
@@ -152,13 +167,14 @@ export class RepoCounterService {
   }
 }
 
-async function countDirectory(rootDir: string): Promise<{
+async function countDirectory(rootDir: string, config: RuntimeConfig): Promise<{
   fileCount: number;
   languages: LanguageStat[];
   lineCount: number;
 }> {
   let fileCount = 0;
   let lineCount = 0;
+  let scannedBytes = 0;
   const languages = new Map<string, LanguageStat>();
   const stack = [rootDir];
 
@@ -180,11 +196,28 @@ async function countDirectory(rootDir: string): Promise<{
         continue;
       }
 
-      if (!entry.isFile() || !(await isTextFile(fullPath))) {
+      if (!entry.isFile()) {
         continue;
       }
 
-      if (shouldIgnoreCountFile(fullPath)) {
+      const relativePath = path.relative(rootDir, fullPath);
+      if (shouldIgnoreCountFile(relativePath)) {
+        continue;
+      }
+
+      const fileStats = await stat(fullPath);
+      if (fileStats.size > config.maxFileSizeBytes) {
+        throw new HttpError(413, `File exceeds ${Math.round(config.maxFileSizeBytes / 1024 / 1024)} MB limit`);
+      }
+      if (++fileCount > config.maxFilesPerScan) {
+        throw new HttpError(413, `Repository exceeds ${config.maxFilesPerScan} file limit`);
+      }
+      scannedBytes += fileStats.size;
+      if (scannedBytes > config.maxScanBytes) {
+        throw new HttpError(413, `Repository exceeds ${Math.round(config.maxScanBytes / 1024 / 1024)} MB scan limit`);
+      }
+      if (!(await isTextFile(fullPath))) {
+        fileCount -= 1;
         continue;
       }
 
@@ -196,7 +229,6 @@ async function countDirectory(rootDir: string): Promise<{
       current.lines += lines;
       languages.set(language, current);
 
-      fileCount += 1;
       lineCount += lines;
     }
   }
@@ -209,8 +241,14 @@ async function countDirectory(rootDir: string): Promise<{
 }
 
 async function isTextFile(filePath: string): Promise<boolean> {
-  const buffer = await readFile(filePath);
-  return !buffer.subarray(0, 4096).includes(0);
+  const handle = await open(filePath, "r");
+  try {
+    const buffer = Buffer.alloc(4096);
+    const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+    return !buffer.subarray(0, bytesRead).includes(0);
+  } finally {
+    await handle.close();
+  }
 }
 
 async function countFileLines(filePath: string): Promise<number> {
@@ -246,11 +284,12 @@ async function runCommand(
   command: string,
   args: string[],
   env: NodeJS.ProcessEnv,
-  timeoutMs: number
-): Promise<void> {
-  await new Promise<void>((resolve, reject) => {
+  timeoutMs: number,
+  captureStdout = false
+): Promise<string> {
+  return new Promise<string>((resolve, reject) => {
     const child = spawn(command, args, { env });
-    let stderr = "";
+    let stdout = "";
     let timedOut = false;
 
     const timeout = setTimeout(() => {
@@ -258,9 +297,12 @@ async function runCommand(
       child.kill("SIGTERM");
     }, timeoutMs);
 
-    child.stderr.on("data", (chunk: Buffer | string) => {
-      stderr += chunk.toString();
-    });
+    if (captureStdout) {
+      child.stdout.on("data", (chunk: Buffer | string) => {
+        stdout += chunk.toString();
+      });
+    }
+    child.stderr.resume();
 
     child.on("error", (error) => {
       clearTimeout(timeout);
@@ -271,7 +313,7 @@ async function runCommand(
       clearTimeout(timeout);
 
       if (code === 0) {
-        resolve();
+        resolve(stdout);
         return;
       }
 
@@ -280,7 +322,36 @@ async function runCommand(
         return;
       }
 
-      reject(new HttpError(400, stderr.trim() || `${command} failed with code ${code}`));
+      reject(new HttpError(400, `${command} failed`));
     });
   });
 }
+
+function getGitHost(repo: string): string {
+  if (/^[a-zA-Z][a-zA-Z\d+.-]*:\/\//.test(repo)) {
+    let url: URL;
+    try {
+      url = new URL(repo);
+    } catch {
+      throw new HttpError(400, "Invalid repository URL");
+    }
+    if (!["https:", "ssh:"].includes(url.protocol) || url.username && url.protocol === "https:") {
+      throw new HttpError(400, "Only credential-free HTTPS and SSH repository URLs are allowed");
+    }
+    return url.hostname.toLowerCase();
+  }
+
+  const scpMatch = repo.match(/^[a-zA-Z0-9._-]+@([a-zA-Z0-9.-]+):[^\s]+$/);
+  if (!scpMatch) {
+    throw new HttpError(400, "Invalid repository URL");
+  }
+  return scpMatch[1].toLowerCase();
+}
+
+function redactRepoUrl(repo: string): string {
+  return repo.replace(/(https?:\/\/)[^/@\s]+@/i, "$1***@");
+}
+
+function isSshRepository(repo: string): boolean {
+  return repo.startsWith("ssh://") || /^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+:[^\s]+$/.test(repo);
+}

src/types.ts

@@ -5,6 +5,7 @@ export type LanguageStat = {
 };
 
 export type CountResult = {
+  commit: string;
   repo: string;
   ref: string | null;
   sshKey: string | null;
@@ -16,6 +17,16 @@ export type CountResult = {
   durationMs: number;
 };
 
+export type ComparisonResult = {
+  base: CountResult;
+  delta: {
+    fileCount: number;
+    languages: LanguageStat[];
+    lineCount: number;
+  };
+  head: CountResult;
+};
+
 export type CountRequest = {
   repo: string;
   ref: string | null;