Harden scans and add LOC comparison endpoint #1

Merged
luna merged 2 commits from harden-loc-service into main 2026-07-21 19:33:08 +02:00
12 changed files with 279 additions and 29 deletions
Showing only changes of commit cd84c68a3e - Show all commits
+4
View File
@@ -1,10 +1,14 @@
PORT=3000 PORT=3000
API_KEY= API_KEY=
ALLOWED_GIT_HOSTS=gitea.reversed.dev
CACHE_TTL_MINUTES=5 CACHE_TTL_MINUTES=5
CACHE_SWEEP_INTERVAL_MINUTES=5 CACHE_SWEEP_INTERVAL_MINUTES=5
RATE_LIMIT_WINDOW_MINUTES=5 RATE_LIMIT_WINDOW_MINUTES=5
RATE_LIMIT_MAX=30 RATE_LIMIT_MAX=30
MAX_CONCURRENT_SCANS=4 MAX_CONCURRENT_SCANS=4
MAX_FILES_PER_SCAN=20000
MAX_FILE_SIZE_MB=5
MAX_SCAN_SIZE_MB=100
CLONE_TIMEOUT_SECONDS=45 CLONE_TIMEOUT_SECONDS=45
DEFAULT_SSH_KEY_NAME=loc_via_git_ed25519 DEFAULT_SSH_KEY_NAME=loc_via_git_ed25519
GENERATE_SSH_KEY_IF_MISSING=false GENERATE_SSH_KEY_IF_MISSING=false
+14 -2
View File
@@ -11,6 +11,7 @@ Tiny API that clones a Git repo and counts its non-empty lines of code.
- Generic project metadata files are ignored during counting - Generic project metadata files are ignored during counting
- Basic rate limiting - Basic rate limiting
- Bounded concurrent scans so the host does not get hammered - Bounded concurrent scans so the host does not get hammered
- Host allowlist and scan/file size limits, with a capped temporary filesystem in Docker
- SSH key file support for private repos - SSH key file support for private repos
- Docker Compose deployment - Docker Compose deployment
@@ -19,7 +20,9 @@ Tiny API that clones a Git repo and counts its non-empty lines of code.
- `GET /loc.txt?repo=<git-url>&ssh_key=<optional-key-file>&ref=<optional-ref>&api_key=<optional-api-key>` - `GET /loc.txt?repo=<git-url>&ssh_key=<optional-key-file>&ref=<optional-ref>&api_key=<optional-api-key>`
- Returns the line count as plain text. - Returns the line count as plain text.
- `GET /loc?repo=<git-url>&ssh_key=<optional-key-file>&ref=<optional-ref>&api_key=<optional-api-key>` - `GET /loc?repo=<git-url>&ssh_key=<optional-key-file>&ref=<optional-ref>&api_key=<optional-api-key>`
- Returns JSON metadata, including a language breakdown by files and non-empty lines. - Returns JSON metadata, including the resolved commit SHA and a language breakdown by files and non-empty lines.
- `GET /loc/diff?repo=<git-url>&base=<git-ref>&head=<git-ref>&ssh_key=<optional-key-file>`
- Returns both snapshots and their total/per-language LOC delta.
- `GET /health` - `GET /health`
- Health plus queue/cache stats. - Health plus queue/cache stats.
@@ -71,7 +74,12 @@ curl "http://localhost:3000/ssh/public-key?ssh_key=loc_via_git_ed25519"
- Only text files are counted. - Only text files are counted.
- Empty lines are ignored. - Empty lines are ignored.
- Generic project metadata files are skipped with a filename blacklist, for example `package.json`, lockfiles, `tsconfig.json`, and similar config/build files. - Generic project metadata, generated/minified files, and common build/vendor directories are skipped.
- Scans are rejected when they exceed the configured file, per-file, or total scanned-byte limits.
## Repository access
`ALLOWED_GIT_HOSTS` is required and accepts a comma-separated host allowlist, such as `gitea.reversed.dev,github.com`. Set it to `*` only if you explicitly accept arbitrary repository hosts. Local paths, `file://` URLs, non-SSH/HTTPS protocols, and HTTPS URLs containing credentials are rejected.
## Configuration ## Configuration
@@ -80,11 +88,15 @@ Copy `.env.example` to `.env` and adjust:
```env ```env
PORT=3000 PORT=3000
API_KEY= API_KEY=
ALLOWED_GIT_HOSTS=gitea.reversed.dev
CACHE_TTL_MINUTES=5 CACHE_TTL_MINUTES=5
CACHE_SWEEP_INTERVAL_MINUTES=5 CACHE_SWEEP_INTERVAL_MINUTES=5
RATE_LIMIT_WINDOW_MINUTES=5 RATE_LIMIT_WINDOW_MINUTES=5
RATE_LIMIT_MAX=30 RATE_LIMIT_MAX=30
MAX_CONCURRENT_SCANS=4 MAX_CONCURRENT_SCANS=4
MAX_FILES_PER_SCAN=20000
MAX_FILE_SIZE_MB=5
MAX_SCAN_SIZE_MB=100
CLONE_TIMEOUT_SECONDS=45 CLONE_TIMEOUT_SECONDS=45
DEFAULT_SSH_KEY_NAME=loc_via_git_ed25519 DEFAULT_SSH_KEY_NAME=loc_via_git_ed25519
GENERATE_SSH_KEY_IF_MISSING=false GENERATE_SSH_KEY_IF_MISSING=false
+5 -1
View File
@@ -14,11 +14,15 @@ services:
environment: environment:
PORT: 3000 PORT: 3000
API_KEY: ${API_KEY:-} API_KEY: ${API_KEY:-}
ALLOWED_GIT_HOSTS: ${ALLOWED_GIT_HOSTS:-gitea.reversed.dev}
CACHE_TTL_MINUTES: ${CACHE_TTL_MINUTES:-5} CACHE_TTL_MINUTES: ${CACHE_TTL_MINUTES:-5}
CACHE_SWEEP_INTERVAL_MINUTES: ${CACHE_SWEEP_INTERVAL_MINUTES:-5} CACHE_SWEEP_INTERVAL_MINUTES: ${CACHE_SWEEP_INTERVAL_MINUTES:-5}
RATE_LIMIT_WINDOW_MINUTES: ${RATE_LIMIT_WINDOW_MINUTES:-5} RATE_LIMIT_WINDOW_MINUTES: ${RATE_LIMIT_WINDOW_MINUTES:-5}
RATE_LIMIT_MAX: ${RATE_LIMIT_MAX:-30} RATE_LIMIT_MAX: ${RATE_LIMIT_MAX:-30}
MAX_CONCURRENT_SCANS: ${MAX_CONCURRENT_SCANS:-4} MAX_CONCURRENT_SCANS: ${MAX_CONCURRENT_SCANS:-4}
MAX_FILES_PER_SCAN: ${MAX_FILES_PER_SCAN:-20000}
MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-5}
MAX_SCAN_SIZE_MB: ${MAX_SCAN_SIZE_MB:-100}
CLONE_TIMEOUT_SECONDS: ${CLONE_TIMEOUT_SECONDS:-45} CLONE_TIMEOUT_SECONDS: ${CLONE_TIMEOUT_SECONDS:-45}
DEFAULT_SSH_KEY_NAME: ${DEFAULT_SSH_KEY_NAME:-loc_via_git_ed25519} DEFAULT_SSH_KEY_NAME: ${DEFAULT_SSH_KEY_NAME:-loc_via_git_ed25519}
GENERATE_SSH_KEY_IF_MISSING: ${GENERATE_SSH_KEY_IF_MISSING:-false} GENERATE_SSH_KEY_IF_MISSING: ${GENERATE_SSH_KEY_IF_MISSING:-false}
@@ -28,7 +32,7 @@ services:
volumes: volumes:
- ssh_keys:/app/keys - ssh_keys:/app/keys
tmpfs: tmpfs:
- /tmp/loc-via-git - /tmp/loc-via-git:size=${TMPFS_SIZE:-512m},mode=1777
restart: unless-stopped restart: unless-stopped
volumes: volumes:
+39 -1
View File
@@ -4,7 +4,7 @@ import rateLimit from "express-rate-limit";
import type { RuntimeConfig } from "./config.js"; import type { RuntimeConfig } from "./config.js";
import { HttpError } from "./errors.js"; import { HttpError } from "./errors.js";
import { asyncHandler, readOptionalString } from "./lib/http.js"; import { asyncHandler, readOptionalString } from "./lib/http.js";
import type { CountRequest } from "./types.js"; import type { ComparisonResult, CountRequest } from "./types.js";
type AppDependencies = { type AppDependencies = {
getHealth: () => Record<string, number>; getHealth: () => Record<string, number>;
@@ -62,6 +62,17 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
res.json(result); res.json(result);
})); }));
app.get("/loc/diff", asyncHandler(async (req, res) => {
const base = readRequiredString(req.query.base, "base");
const head = readRequiredString(req.query.head, "head");
const shared = readCountRequest(req);
const [baseResult, headResult] = await Promise.all([
deps.count({ ...shared, ref: base }),
deps.count({ ...shared, ref: head })
]);
res.json(createComparison(baseResult, headResult));
}));
app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => { app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => {
const message = error instanceof Error ? error.message : "Unknown error"; const message = error instanceof Error ? error.message : "Unknown error";
const status = error instanceof HttpError ? error.statusCode : message.startsWith("Missing ") ? 400 : 500; const status = error instanceof HttpError ? error.statusCode : message.startsWith("Missing ") ? 400 : 500;
@@ -71,6 +82,25 @@ export function createApp(config: RuntimeConfig, deps: AppDependencies) {
return app; return app;
} }
function createComparison(base: import("./types.js").CountResult, head: import("./types.js").CountResult): ComparisonResult {
const languages = new Map(base.languages.map((language) => [language.language, { ...language, files: -language.files, lines: -language.lines }]));
for (const language of head.languages) {
const delta = languages.get(language.language) ?? { language: language.language, files: 0, lines: 0 };
delta.files += language.files;
delta.lines += language.lines;
languages.set(language.language, delta);
}
return {
base,
head,
delta: {
fileCount: head.fileCount - base.fileCount,
lineCount: head.lineCount - base.lineCount,
languages: Array.from(languages.values()).filter((language) => language.files !== 0 || language.lines !== 0).sort((a, b) => b.lines - a.lines || a.language.localeCompare(b.language))
}
};
}
function readCountRequest(req: Request): CountRequest { function readCountRequest(req: Request): CountRequest {
const repo = typeof req.query.repo === "string" ? req.query.repo.trim() : ""; const repo = typeof req.query.repo === "string" ? req.query.repo.trim() : "";
if (!repo) { if (!repo) {
@@ -83,3 +113,11 @@ function readCountRequest(req: Request): CountRequest {
sshKey: readOptionalString(req.query.ssh_key) sshKey: readOptionalString(req.query.ssh_key)
}; };
} }
function readRequiredString(value: unknown, name: string): string {
const result = readOptionalString(value);
if (!result) {
throw new HttpError(400, `Missing ${name} query parameter`);
}
return result;
}
+13
View File
@@ -2,6 +2,7 @@ import path from "node:path";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
export type RuntimeConfig = { export type RuntimeConfig = {
allowedGitHosts: string[];
apiKey: string; apiKey: string;
cacheSweepIntervalMs: number; cacheSweepIntervalMs: number;
cacheTtlMs: number; cacheTtlMs: number;
@@ -9,6 +10,9 @@ export type RuntimeConfig = {
defaultSshKeyName: string; defaultSshKeyName: string;
generateSshKeyIfMissing: boolean; generateSshKeyIfMissing: boolean;
maxConcurrentScans: number; maxConcurrentScans: number;
maxFilesPerScan: number;
maxFileSizeBytes: number;
maxScanBytes: number;
port: number; port: number;
rateLimitMax: number; rateLimitMax: number;
rateLimitWindowMs: number; rateLimitWindowMs: number;
@@ -19,6 +23,7 @@ export type RuntimeConfig = {
export function loadConfig(): RuntimeConfig { export function loadConfig(): RuntimeConfig {
return { return {
allowedGitHosts: readEnvList("ALLOWED_GIT_HOSTS"),
apiKey: readEnvString("API_KEY", ""), apiKey: readEnvString("API_KEY", ""),
cacheSweepIntervalMs: readEnvMinutes("CACHE_SWEEP_INTERVAL_MINUTES", 5), cacheSweepIntervalMs: readEnvMinutes("CACHE_SWEEP_INTERVAL_MINUTES", 5),
cacheTtlMs: readEnvMinutes("CACHE_TTL_MINUTES", 5), cacheTtlMs: readEnvMinutes("CACHE_TTL_MINUTES", 5),
@@ -26,6 +31,9 @@ export function loadConfig(): RuntimeConfig {
defaultSshKeyName: readEnvString("DEFAULT_SSH_KEY_NAME", "loc_via_git_ed25519"), defaultSshKeyName: readEnvString("DEFAULT_SSH_KEY_NAME", "loc_via_git_ed25519"),
generateSshKeyIfMissing: readEnvBoolean("GENERATE_SSH_KEY_IF_MISSING", false), generateSshKeyIfMissing: readEnvBoolean("GENERATE_SSH_KEY_IF_MISSING", false),
maxConcurrentScans: readEnvNumber("MAX_CONCURRENT_SCANS", 4), maxConcurrentScans: readEnvNumber("MAX_CONCURRENT_SCANS", 4),
maxFilesPerScan: readEnvNumber("MAX_FILES_PER_SCAN", 20_000),
maxFileSizeBytes: readEnvNumber("MAX_FILE_SIZE_MB", 5) * 1024 * 1024,
maxScanBytes: readEnvNumber("MAX_SCAN_SIZE_MB", 100) * 1024 * 1024,
port: readEnvNumber("PORT", 3000), port: readEnvNumber("PORT", 3000),
rateLimitMax: readEnvNumber("RATE_LIMIT_MAX", 30), rateLimitMax: readEnvNumber("RATE_LIMIT_MAX", 30),
rateLimitWindowMs: readEnvMinutes("RATE_LIMIT_WINDOW_MINUTES", 5), rateLimitWindowMs: readEnvMinutes("RATE_LIMIT_WINDOW_MINUTES", 5),
@@ -35,6 +43,11 @@ export function loadConfig(): RuntimeConfig {
}; };
} }
function readEnvList(name: string): string[] {
const raw = process.env[name]?.trim() ?? "";
return raw.split(",").map((value) => value.trim().toLowerCase()).filter(Boolean);
}
function readEnvMinutes(name: string, fallback: number): number { function readEnvMinutes(name: string, fallback: number): number {
return readEnvNumber(name, fallback) * 60_000; return readEnvNumber(name, fallback) * 60_000;
} }
+12 -1
View File
@@ -51,7 +51,18 @@ const ignoredFileNames = new Set([
"yarn.lock" "yarn.lock"
]); ]);
const ignoredDirectoryNames = new Set([
".angular", ".cache", ".next", ".nuxt", ".output", ".parcel-cache", ".svelte-kit",
".terraform", ".venv", "bower_components", "build", "coverage", "dist", "node_modules",
"out", "pods", "target", "vendor"
]);
export function shouldIgnoreCountFile(filePath: string): boolean { export function shouldIgnoreCountFile(filePath: string): boolean {
const fileName = path.basename(filePath).toLowerCase(); const fileName = path.basename(filePath).toLowerCase();
return ignoredFileNames.has(fileName); const directories = path.dirname(filePath).toLowerCase().split(path.sep);
return ignoredFileNames.has(fileName)
|| directories.some((directory) => ignoredDirectoryNames.has(directory))
|| /\.(generated|designer|g|pb)\.[^.]+$/.test(fileName)
|| /\.min\.(css|js|mjs|cjs)$/.test(fileName)
|| fileName.endsWith(".snap");
} }
+42 -1
View File
@@ -1,6 +1,44 @@
import path from "node:path"; import path from "node:path";
const extensionToLanguage: Record<string, string> = { const extensionToLanguage: Record<string, string> = {
".astro": "Astro",
".bash": "Shell",
".bat": "Batchfile",
".cjs": "JavaScript",
".clj": "Clojure",
".cmake": "CMake",
".coffee": "CoffeeScript",
".dart": "Dart",
".dockerfile": "Dockerfile",
".ex": "Elixir",
".exs": "Elixir",
".fish": "Fish",
".fs": "F#",
".fsx": "F#",
".gd": "GDScript",
".graphql": "GraphQL",
".groovy": "Groovy",
".hs": "Haskell",
".ini": "INI",
".ipynb": "Jupyter Notebook",
".jsx": "JavaScript React",
".less": "Less",
".lock": "Lockfile",
".m": "Objective-C",
".nim": "Nim",
".pl": "Perl",
".proto": "Protocol Buffers",
".ps1": "PowerShell",
".r": "R",
".rkt": "Racket",
".scala": "Scala",
".sol": "Solidity",
".styl": "Stylus",
".tf": "Terraform",
".v": "Verilog",
".vala": "Vala",
".wasm": "WebAssembly",
".zig": "Zig",
".c": "C", ".c": "C",
".cc": "C++", ".cc": "C++",
".cpp": "C++", ".cpp": "C++",
@@ -13,7 +51,6 @@ const extensionToLanguage: Record<string, string> = {
".java": "Java", ".java": "Java",
".js": "JavaScript", ".js": "JavaScript",
".json": "JSON", ".json": "JSON",
".jsx": "JavaScript React",
".kt": "Kotlin", ".kt": "Kotlin",
".lua": "Lua", ".lua": "Lua",
".md": "Markdown", ".md": "Markdown",
@@ -46,6 +83,10 @@ export function detectLanguage(filePath: string): string {
return "Dockerfile"; return "Dockerfile";
} }
if (["makefile", "gnumakefile"].includes(fileName)) {
return "Makefile";
}
if (fileName.endsWith(".d.ts")) { if (fileName.endsWith(".d.ts")) {
return "TypeScript"; return "TypeScript";
} }
+96 -22
View File
@@ -1,6 +1,6 @@
import { spawn } from "node:child_process"; import { spawn } from "node:child_process";
import { createReadStream } from "node:fs"; import { createReadStream } from "node:fs";
import { mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises"; import { mkdir, mkdtemp, open, readdir, rm, stat } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import type { RuntimeConfig } from "../config.js"; import type { RuntimeConfig } from "../config.js";
@@ -51,7 +51,8 @@ export class RepoCounterService {
} }
async count(request: CountRequest): Promise<CountResult> { async count(request: CountRequest): Promise<CountResult> {
const cacheKey = JSON.stringify(request); const validatedRequest = this.validateRequest(request);
const cacheKey = JSON.stringify(validatedRequest);
this.clearExpiredCache(); this.clearExpiredCache();
const cached = this.cache.get(cacheKey); const cached = this.cache.get(cacheKey);
@@ -66,7 +67,7 @@ export class RepoCounterService {
} }
const task = this.semaphore.use(async () => { const task = this.semaphore.use(async () => {
const value = await this.cloneAndCount(request); const value = await this.cloneAndCount(validatedRequest);
this.cache.set(cacheKey, { this.cache.set(cacheKey, {
value, value,
expiresAt: Date.now() + this.config.cacheTtlMs expiresAt: Date.now() + this.config.cacheTtlMs
@@ -90,10 +91,14 @@ export class RepoCounterService {
try { try {
await this.runGitClone(request.repo, repoDir, request.ref, request.sshKey); await this.runGitClone(request.repo, repoDir, request.ref, request.sshKey);
const stats = await countDirectory(repoDir); const [stats, commit] = await Promise.all([
countDirectory(repoDir, this.config),
runCommand("git", ["-C", repoDir, "rev-parse", "HEAD"], { ...process.env }, this.config.cloneTimeoutMs, true)
]);
return { return {
repo: request.repo, commit: commit.trim(),
repo: redactRepoUrl(request.repo),
ref: request.ref, ref: request.ref,
sshKey: request.sshKey, sshKey: request.sshKey,
lineCount: stats.lineCount, lineCount: stats.lineCount,
@@ -108,14 +113,16 @@ export class RepoCounterService {
} }
private async runGitClone(repo: string, targetDir: string, ref: string | null, sshKey: string | null): Promise<void> { private async runGitClone(repo: string, targetDir: string, ref: string | null, sshKey: string | null): Promise<void> {
const env = { ...process.env }; const env: NodeJS.ProcessEnv = { ...process.env, GIT_TERMINAL_PROMPT: "0" };
const keyPath = await this.keyManager.resolvePrivateKeyPath(sshKey); const keyPath = sshKey || isSshRepository(repo)
? await this.keyManager.resolvePrivateKeyPath(sshKey)
: null;
if (keyPath) { if (keyPath) {
env.GIT_SSH_COMMAND = `ssh -i "${keyPath}" -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new`; env.GIT_SSH_COMMAND = `ssh -i "${keyPath}" -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new`;
} }
await runCommand("git", ["clone", "--depth", "1", "--single-branch", repo, targetDir], env, this.config.cloneTimeoutMs); await runCommand("git", ["-c", "protocol.file.allow=never", "clone", "--depth", "1", "--single-branch", repo, targetDir], env, this.config.cloneTimeoutMs);
if (ref) { if (ref) {
await runCommand("git", ["-C", targetDir, "fetch", "--depth", "1", "origin", ref], env, this.config.cloneTimeoutMs); await runCommand("git", ["-C", targetDir, "fetch", "--depth", "1", "origin", ref], env, this.config.cloneTimeoutMs);
@@ -123,6 +130,17 @@ export class RepoCounterService {
} }
} }
private validateRequest(request: CountRequest): CountRequest {
const host = getGitHost(request.repo);
if (this.config.allowedGitHosts.length === 0) {
throw new HttpError(503, "Git hosts are not configured");
}
if (!this.config.allowedGitHosts.includes("*") && !this.config.allowedGitHosts.includes(host)) {
throw new HttpError(403, "Git host is not allowed");
}
return request;
}
private async cleanupStaleTempDirs(): Promise<void> { private async cleanupStaleTempDirs(): Promise<void> {
const entries = await readdir(this.config.tempRoot, { withFileTypes: true }).catch(() => []); const entries = await readdir(this.config.tempRoot, { withFileTypes: true }).catch(() => []);
const staleBefore = Date.now() - this.config.cloneTimeoutMs * 2; const staleBefore = Date.now() - this.config.cloneTimeoutMs * 2;
@@ -152,13 +170,14 @@ export class RepoCounterService {
} }
} }
async function countDirectory(rootDir: string): Promise<{ async function countDirectory(rootDir: string, config: RuntimeConfig): Promise<{
fileCount: number; fileCount: number;
languages: LanguageStat[]; languages: LanguageStat[];
lineCount: number; lineCount: number;
}> { }> {
let fileCount = 0; let fileCount = 0;
let lineCount = 0; let lineCount = 0;
let scannedBytes = 0;
const languages = new Map<string, LanguageStat>(); const languages = new Map<string, LanguageStat>();
const stack = [rootDir]; const stack = [rootDir];
@@ -180,11 +199,28 @@ async function countDirectory(rootDir: string): Promise<{
continue; continue;
} }
if (!entry.isFile() || !(await isTextFile(fullPath))) { if (!entry.isFile()) {
continue; continue;
} }
if (shouldIgnoreCountFile(fullPath)) { const relativePath = path.relative(rootDir, fullPath);
if (shouldIgnoreCountFile(relativePath)) {
continue;
}
const fileStats = await stat(fullPath);
if (fileStats.size > config.maxFileSizeBytes) {
throw new HttpError(413, `File exceeds ${Math.round(config.maxFileSizeBytes / 1024 / 1024)} MB limit`);
}
if (++fileCount > config.maxFilesPerScan) {
throw new HttpError(413, `Repository exceeds ${config.maxFilesPerScan} file limit`);
}
scannedBytes += fileStats.size;
if (scannedBytes > config.maxScanBytes) {
throw new HttpError(413, `Repository exceeds ${Math.round(config.maxScanBytes / 1024 / 1024)} MB scan limit`);
}
if (!(await isTextFile(fullPath))) {
fileCount -= 1;
continue; continue;
} }
@@ -196,7 +232,6 @@ async function countDirectory(rootDir: string): Promise<{
current.lines += lines; current.lines += lines;
languages.set(language, current); languages.set(language, current);
fileCount += 1;
lineCount += lines; lineCount += lines;
} }
} }
@@ -209,8 +244,14 @@ async function countDirectory(rootDir: string): Promise<{
} }
async function isTextFile(filePath: string): Promise<boolean> { async function isTextFile(filePath: string): Promise<boolean> {
const buffer = await readFile(filePath); const handle = await open(filePath, "r");
return !buffer.subarray(0, 4096).includes(0); try {
const buffer = Buffer.alloc(4096);
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
return !buffer.subarray(0, bytesRead).includes(0);
} finally {
await handle.close();
}
} }
async function countFileLines(filePath: string): Promise<number> { async function countFileLines(filePath: string): Promise<number> {
@@ -246,11 +287,12 @@ async function runCommand(
command: string, command: string,
args: string[], args: string[],
env: NodeJS.ProcessEnv, env: NodeJS.ProcessEnv,
timeoutMs: number timeoutMs: number,
): Promise<void> { captureStdout = false
await new Promise<void>((resolve, reject) => { ): Promise<string> {
return new Promise<string>((resolve, reject) => {
const child = spawn(command, args, { env }); const child = spawn(command, args, { env });
let stderr = ""; let stdout = "";
let timedOut = false; let timedOut = false;
const timeout = setTimeout(() => { const timeout = setTimeout(() => {
@@ -258,9 +300,12 @@ async function runCommand(
child.kill("SIGTERM"); child.kill("SIGTERM");
}, timeoutMs); }, timeoutMs);
child.stderr.on("data", (chunk: Buffer | string) => { if (captureStdout) {
stderr += chunk.toString(); child.stdout.on("data", (chunk: Buffer | string) => {
stdout += chunk.toString();
}); });
}
child.stderr.resume();
child.on("error", (error) => { child.on("error", (error) => {
clearTimeout(timeout); clearTimeout(timeout);
@@ -271,7 +316,7 @@ async function runCommand(
clearTimeout(timeout); clearTimeout(timeout);
if (code === 0) { if (code === 0) {
resolve(); resolve(stdout);
return; return;
} }
@@ -280,7 +325,36 @@ async function runCommand(
return; return;
} }
reject(new HttpError(400, stderr.trim() || `${command} failed with code ${code}`)); reject(new HttpError(400, `${command} failed`));
}); });
}); });
} }
function getGitHost(repo: string): string {
if (/^[a-zA-Z][a-zA-Z\d+.-]*:\/\//.test(repo)) {
let url: URL;
try {
url = new URL(repo);
} catch {
throw new HttpError(400, "Invalid repository URL");
}
if (!["https:", "ssh:"].includes(url.protocol) || url.username && url.protocol === "https:") {
throw new HttpError(400, "Only credential-free HTTPS and SSH repository URLs are allowed");
}
return url.hostname.toLowerCase();
}
const scpMatch = repo.match(/^[a-zA-Z0-9._-]+@([a-zA-Z0-9.-]+):[^\s]+$/);
if (!scpMatch) {
throw new HttpError(400, "Invalid repository URL");
}
return scpMatch[1].toLowerCase();
}
function redactRepoUrl(repo: string): string {
return repo.replace(/(https?:\/\/)[^/@\s]+@/i, "$1***@");
}
function isSshRepository(repo: string): boolean {
return repo.startsWith("ssh://") || /^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+:[^\s]+$/.test(repo);
}
+11
View File
@@ -5,6 +5,7 @@ export type LanguageStat = {
}; };
export type CountResult = { export type CountResult = {
commit: string;
repo: string; repo: string;
ref: string | null; ref: string | null;
sshKey: string | null; sshKey: string | null;
@@ -16,6 +17,16 @@ export type CountResult = {
durationMs: number; durationMs: number;
}; };
export type ComparisonResult = {
base: CountResult;
delta: {
fileCount: number;
languages: LanguageStat[];
lineCount: number;
};
head: CountResult;
};
export type CountRequest = { export type CountRequest = {
repo: string; repo: string;
ref: string | null; ref: string | null;
+37
View File
@@ -5,6 +5,7 @@ import request from "supertest";
import { createApp } from "../dist/app.js"; import { createApp } from "../dist/app.js";
const baseConfig = { const baseConfig = {
allowedGitHosts: ["example.com"],
apiKey: "", apiKey: "",
cacheSweepIntervalMs: 300000, cacheSweepIntervalMs: 300000,
cacheTtlMs: 300000, cacheTtlMs: 300000,
@@ -12,6 +13,9 @@ const baseConfig = {
defaultSshKeyName: "loc_via_git_ed25519", defaultSshKeyName: "loc_via_git_ed25519",
generateSshKeyIfMissing: false, generateSshKeyIfMissing: false,
maxConcurrentScans: 4, maxConcurrentScans: 4,
maxFilesPerScan: 20000,
maxFileSizeBytes: 5 * 1024 * 1024,
maxScanBytes: 100 * 1024 * 1024,
port: 3000, port: 3000,
rateLimitMax: 30, rateLimitMax: 30,
rateLimitWindowMs: 300000, rateLimitWindowMs: 300000,
@@ -26,6 +30,7 @@ test("GET /loc returns count metadata", async () => {
getPublicKey: async () => "ssh-ed25519 AAAA", getPublicKey: async () => "ssh-ed25519 AAAA",
getDefaultKeyName: () => "loc_via_git_ed25519", getDefaultKeyName: () => "loc_via_git_ed25519",
count: async (requestInput) => ({ count: async (requestInput) => ({
commit: "abc123",
repo: requestInput.repo, repo: requestInput.repo,
ref: requestInput.ref, ref: requestInput.ref,
sshKey: requestInput.sshKey, sshKey: requestInput.sshKey,
@@ -82,6 +87,7 @@ test("GET /loc accepts api_key query param as an unsafe fallback", async () => {
getPublicKey: async () => "ssh-ed25519 AAAA", getPublicKey: async () => "ssh-ed25519 AAAA",
getDefaultKeyName: () => "loc_via_git_ed25519", getDefaultKeyName: () => "loc_via_git_ed25519",
count: async (requestInput) => ({ count: async (requestInput) => ({
commit: "abc123",
repo: requestInput.repo, repo: requestInput.repo,
ref: requestInput.ref, ref: requestInput.ref,
sshKey: requestInput.sshKey, sshKey: requestInput.sshKey,
@@ -99,3 +105,34 @@ test("GET /loc accepts api_key query param as an unsafe fallback", async () => {
assert.equal(response.status, 200); assert.equal(response.status, 200);
assert.equal(response.body.lineCount, 5); assert.equal(response.body.lineCount, 5);
}); });
test("GET /loc/diff returns aggregate and language deltas", async () => {
const app = createApp(baseConfig, {
getHealth: () => ({ cacheEntries: 0, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }),
getPublicKey: async () => "ssh-ed25519 AAAA",
getDefaultKeyName: () => "loc_via_git_ed25519",
count: async (requestInput) => ({
commit: requestInput.ref === "base" ? "base123" : "head456",
repo: requestInput.repo,
ref: requestInput.ref,
sshKey: requestInput.sshKey,
cached: false,
lineCount: requestInput.ref === "base" ? 10 : 17,
fileCount: requestInput.ref === "base" ? 2 : 3,
languages: requestInput.ref === "base"
? [{ language: "TypeScript", files: 2, lines: 10 }]
: [{ language: "TypeScript", files: 2, lines: 12 }, { language: "Python", files: 1, lines: 5 }],
scannedAt: "2026-01-01T00:00:00.000Z",
durationMs: 10
})
});
const response = await request(app).get("/loc/diff?repo=https://example.com/repo.git&base=base&head=head");
assert.equal(response.status, 200);
assert.equal(response.body.delta.lineCount, 7);
assert.deepEqual(response.body.delta.languages, [
{ language: "Python", files: 1, lines: 5 },
{ language: "TypeScript", files: 0, lines: 2 }
]);
});
+3
View File
@@ -9,4 +9,7 @@ test("shouldIgnoreCountFile skips generic project metadata files", () => {
assert.equal(shouldIgnoreCountFile("/tmp/docker-compose.yml"), true); assert.equal(shouldIgnoreCountFile("/tmp/docker-compose.yml"), true);
assert.equal(shouldIgnoreCountFile("/tmp/src/index.ts"), false); assert.equal(shouldIgnoreCountFile("/tmp/src/index.ts"), false);
assert.equal(shouldIgnoreCountFile("/tmp/README.md"), false); assert.equal(shouldIgnoreCountFile("/tmp/README.md"), false);
assert.equal(shouldIgnoreCountFile("dist/bundle.min.js"), true);
assert.equal(shouldIgnoreCountFile("vendor/lib/index.ts"), true);
assert.equal(shouldIgnoreCountFile("src/api.generated.ts"), true);
}); });
+2
View File
@@ -8,4 +8,6 @@ test("detectLanguage recognizes common source files", () => {
assert.equal(detectLanguage("/tmp/Dockerfile"), "Dockerfile"); assert.equal(detectLanguage("/tmp/Dockerfile"), "Dockerfile");
assert.equal(detectLanguage("/tmp/types.d.ts"), "TypeScript"); assert.equal(detectLanguage("/tmp/types.d.ts"), "TypeScript");
assert.equal(detectLanguage("/tmp/file.unknown"), "Plain Text"); assert.equal(detectLanguage("/tmp/file.unknown"), "Plain Text");
assert.equal(detectLanguage("/tmp/component.astro"), "Astro");
assert.equal(detectLanguage("/tmp/Makefile"), "Makefile");
}); });