Luna
+84
@@ -0,0 +1,84 @@
|
||||
import express, { type NextFunction, type Request, type Response } from "express";
|
||||
import rateLimit from "express-rate-limit";
|
||||
|
||||
import type { RuntimeConfig } from "./config.js";
|
||||
import { HttpError } from "./errors.js";
|
||||
import { asyncHandler, readOptionalString } from "./lib/http.js";
|
||||
import type { CountRequest } from "./types.js";
|
||||
|
||||
type AppDependencies = {
|
||||
getHealth: () => Record<string, number>;
|
||||
getPublicKey: (name: string | null) => Promise<string>;
|
||||
getDefaultKeyName: () => string;
|
||||
count: (request: CountRequest) => Promise<import("./types.js").CountResult>;
|
||||
};
|
||||
|
||||
export function createApp(config: RuntimeConfig, deps: AppDependencies) {
|
||||
const app = express();
|
||||
app.set("trust proxy", config.trustProxy);
|
||||
|
||||
app.use(rateLimit({
|
||||
windowMs: config.rateLimitWindowMs,
|
||||
limit: config.rateLimitMax,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false
|
||||
}));
|
||||
|
||||
app.get("/health", (_req, res) => {
|
||||
res.json({ ok: true, ...deps.getHealth() });
|
||||
});
|
||||
|
||||
app.get("/ssh/public-key", asyncHandler(async (req, res) => {
|
||||
const keyName = readOptionalString(req.query.ssh_key) ?? deps.getDefaultKeyName();
|
||||
const publicKey = await deps.getPublicKey(keyName);
|
||||
res.type("text/plain").send(publicKey);
|
||||
}));
|
||||
|
||||
app.use((req, res, next) => {
|
||||
if (!config.apiKey || req.path === "/ssh/public-key") {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
const headerKey = req.header("x-api-key");
|
||||
const bearer = req.header("authorization")?.replace(/^Bearer\s+/i, "").trim();
|
||||
|
||||
if (headerKey === config.apiKey || bearer === config.apiKey) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
res.status(401).json({ error: "Unauthorized" });
|
||||
});
|
||||
|
||||
app.get("/loc.txt", asyncHandler(async (req, res) => {
|
||||
const result = await deps.count(readCountRequest(req));
|
||||
res.type("text/plain").send(String(result.lineCount));
|
||||
}));
|
||||
|
||||
app.get("/loc", asyncHandler(async (req, res) => {
|
||||
const result = await deps.count(readCountRequest(req));
|
||||
res.json(result);
|
||||
}));
|
||||
|
||||
app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
const status = error instanceof HttpError ? error.statusCode : message.startsWith("Missing ") ? 400 : 500;
|
||||
res.status(status).json({ error: message });
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
function readCountRequest(req: Request): CountRequest {
|
||||
const repo = typeof req.query.repo === "string" ? req.query.repo.trim() : "";
|
||||
if (!repo) {
|
||||
throw new HttpError(400, "Missing repo query parameter");
|
||||
}
|
||||
|
||||
return {
|
||||
repo,
|
||||
ref: readOptionalString(req.query.ref),
|
||||
sshKey: readOptionalString(req.query.ssh_key)
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
export type RuntimeConfig = {
|
||||
apiKey: string;
|
||||
cacheSweepIntervalMs: number;
|
||||
cacheTtlMs: number;
|
||||
cloneTimeoutMs: number;
|
||||
defaultSshKeyName: string;
|
||||
generateSshKeyIfMissing: boolean;
|
||||
maxConcurrentScans: number;
|
||||
port: number;
|
||||
rateLimitMax: number;
|
||||
rateLimitWindowMs: number;
|
||||
sshKeysDir: string;
|
||||
tempRoot: string;
|
||||
trustProxy: boolean | number | string;
|
||||
};
|
||||
|
||||
export function loadConfig(): RuntimeConfig {
|
||||
return {
|
||||
apiKey: readEnvString("API_KEY", ""),
|
||||
cacheSweepIntervalMs: readEnvMinutes("CACHE_SWEEP_INTERVAL_MINUTES", 5),
|
||||
cacheTtlMs: readEnvMinutes("CACHE_TTL_MINUTES", 5),
|
||||
cloneTimeoutMs: readEnvSeconds("CLONE_TIMEOUT_SECONDS", 45),
|
||||
defaultSshKeyName: readEnvString("DEFAULT_SSH_KEY_NAME", "loc_via_git_ed25519"),
|
||||
generateSshKeyIfMissing: readEnvBoolean("GENERATE_SSH_KEY_IF_MISSING", false),
|
||||
maxConcurrentScans: readEnvNumber("MAX_CONCURRENT_SCANS", 4),
|
||||
port: readEnvNumber("PORT", 3000),
|
||||
rateLimitMax: readEnvNumber("RATE_LIMIT_MAX", 30),
|
||||
rateLimitWindowMs: readEnvMinutes("RATE_LIMIT_WINDOW_MINUTES", 5),
|
||||
sshKeysDir: readEnvString("SSH_KEYS_DIR", path.resolve(process.cwd(), "keys")),
|
||||
tempRoot: readEnvString("TMP_DIR", path.join(tmpdir(), "loc-via-git")),
|
||||
trustProxy: readEnvTrustProxy("TRUST_PROXY", false)
|
||||
};
|
||||
}
|
||||
|
||||
function readEnvMinutes(name: string, fallback: number): number {
|
||||
return readEnvNumber(name, fallback) * 60_000;
|
||||
}
|
||||
|
||||
function readEnvSeconds(name: string, fallback: number): number {
|
||||
return readEnvNumber(name, fallback) * 1000;
|
||||
}
|
||||
|
||||
function readEnvNumber(name: string, fallback: number): number {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
throw new Error(`Invalid ${name}: ${raw}`);
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readEnvBoolean(name: string, fallback: boolean): boolean {
|
||||
const raw = process.env[name]?.trim().toLowerCase();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
if (["1", "true", "yes", "on"].includes(raw)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (["0", "false", "no", "off"].includes(raw)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
throw new Error(`Invalid ${name}: ${raw}`);
|
||||
}
|
||||
|
||||
function readEnvTrustProxy(name: string, fallback: boolean | number | string): boolean | number | string {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
if (["true", "false", "1", "0", "yes", "no", "on", "off"].includes(raw.toLowerCase())) {
|
||||
return readEnvBoolean(name, Boolean(fallback));
|
||||
}
|
||||
|
||||
const asNumber = Number(raw);
|
||||
if (Number.isInteger(asNumber) && asNumber >= 0) {
|
||||
return asNumber;
|
||||
}
|
||||
|
||||
return raw;
|
||||
}
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
return process.env[name]?.trim() ?? fallback;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
public readonly statusCode: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import path from "node:path";
|
||||
|
||||
const extensionToLanguage: Record<string, string> = {
|
||||
".c": "C",
|
||||
".cc": "C++",
|
||||
".cpp": "C++",
|
||||
".cs": "C#",
|
||||
".css": "CSS",
|
||||
".go": "Go",
|
||||
".h": "C/C++ Header",
|
||||
".hpp": "C++ Header",
|
||||
".html": "HTML",
|
||||
".java": "Java",
|
||||
".js": "JavaScript",
|
||||
".json": "JSON",
|
||||
".jsx": "JavaScript React",
|
||||
".kt": "Kotlin",
|
||||
".lua": "Lua",
|
||||
".md": "Markdown",
|
||||
".mjs": "JavaScript",
|
||||
".php": "PHP",
|
||||
".py": "Python",
|
||||
".rb": "Ruby",
|
||||
".rs": "Rust",
|
||||
".scss": "SCSS",
|
||||
".sh": "Shell",
|
||||
".sql": "SQL",
|
||||
".svg": "SVG",
|
||||
".svelte": "Svelte",
|
||||
".swift": "Swift",
|
||||
".toml": "TOML",
|
||||
".ts": "TypeScript",
|
||||
".tsx": "TypeScript React",
|
||||
".txt": "Plain Text",
|
||||
".vue": "Vue",
|
||||
".xml": "XML",
|
||||
".yaml": "YAML",
|
||||
".yml": "YAML"
|
||||
};
|
||||
|
||||
export function detectLanguage(filePath: string): string {
|
||||
const fileName = path.basename(filePath).toLowerCase();
|
||||
const extension = path.extname(fileName);
|
||||
|
||||
if (fileName === "dockerfile") {
|
||||
return "Dockerfile";
|
||||
}
|
||||
|
||||
if (fileName.endsWith(".d.ts")) {
|
||||
return "TypeScript";
|
||||
}
|
||||
|
||||
return extensionToLanguage[extension] ?? "Plain Text";
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
|
||||
export function asyncHandler(handler: (req: Request, res: Response) => Promise<void>) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
void handler(req, res).catch(next);
|
||||
};
|
||||
}
|
||||
|
||||
export function readOptionalString(value: unknown): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trimmed = value.trim();
|
||||
return trimmed === "" ? null : trimmed;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
export class Semaphore {
|
||||
private current = 0;
|
||||
private readonly waiting: Array<() => void> = [];
|
||||
|
||||
constructor(private readonly limit: number) {}
|
||||
|
||||
get active(): number {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
get queued(): number {
|
||||
return this.waiting.length;
|
||||
}
|
||||
|
||||
async use<T>(task: () => Promise<T>): Promise<T> {
|
||||
await this.acquire();
|
||||
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
this.release();
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<void> {
|
||||
if (this.current < this.limit) {
|
||||
this.current += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
this.waiting.push(() => {
|
||||
this.current += 1;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private release(): void {
|
||||
this.current -= 1;
|
||||
const next = this.waiting.shift();
|
||||
next?.();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { loadConfig } from "./config.js";
|
||||
import { createApp } from "./app.js";
|
||||
import { KeyManager } from "./services/key-manager.js";
|
||||
import { RepoCounterService } from "./services/repo-counter.js";
|
||||
|
||||
const config = loadConfig();
|
||||
const keyManager = new KeyManager(config);
|
||||
const repoCounter = new RepoCounterService(config, keyManager);
|
||||
|
||||
await keyManager.initialize();
|
||||
await repoCounter.initialize();
|
||||
|
||||
const app = createApp(config, {
|
||||
getHealth: () => repoCounter.getHealth(),
|
||||
getPublicKey: (name) => keyManager.getPublicKey(name),
|
||||
getDefaultKeyName: () => keyManager.getDefaultKeyName(),
|
||||
count: (request) => repoCounter.count(request)
|
||||
});
|
||||
|
||||
app.listen(config.port, () => {
|
||||
console.log(`loc-via-git listening on ${config.port}`);
|
||||
});
|
||||
@@ -0,0 +1,118 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir, readFile, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { RuntimeConfig } from "../config.js";
|
||||
import { HttpError } from "../errors.js";
|
||||
|
||||
export class KeyManager {
|
||||
constructor(private readonly config: RuntimeConfig) {}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
await mkdir(this.config.sshKeysDir, { recursive: true });
|
||||
|
||||
if (!this.config.generateSshKeyIfMissing) {
|
||||
return;
|
||||
}
|
||||
|
||||
const privateKeyPath = this.getPrivateKeyPath(this.config.defaultSshKeyName);
|
||||
const publicKeyPath = this.getPublicKeyPath(this.config.defaultSshKeyName);
|
||||
const privateExists = await exists(privateKeyPath);
|
||||
const publicExists = await exists(publicKeyPath);
|
||||
|
||||
if (privateExists && publicExists) {
|
||||
return;
|
||||
}
|
||||
|
||||
await runCommand("ssh-keygen", [
|
||||
"-t", "ed25519",
|
||||
"-N", "",
|
||||
"-f", privateKeyPath,
|
||||
"-C", "loc-via-git"
|
||||
]);
|
||||
}
|
||||
|
||||
async resolvePrivateKeyPath(name: string | null): Promise<string | null> {
|
||||
const keyName = name ?? this.config.defaultSshKeyName;
|
||||
if (!keyName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.assertValidKeyName(keyName);
|
||||
|
||||
const keyPath = this.getPrivateKeyPath(keyName);
|
||||
if (!(await exists(keyPath))) {
|
||||
if (keyName === this.config.defaultSshKeyName && this.config.generateSshKeyIfMissing) {
|
||||
await this.initialize();
|
||||
}
|
||||
}
|
||||
|
||||
if (!(await exists(keyPath))) {
|
||||
throw new HttpError(400, "SSH key not found");
|
||||
}
|
||||
|
||||
return keyPath;
|
||||
}
|
||||
|
||||
async getPublicKey(name: string | null): Promise<string> {
|
||||
const keyName = name ?? this.config.defaultSshKeyName;
|
||||
this.assertValidKeyName(keyName);
|
||||
|
||||
const keyPath = this.getPublicKeyPath(keyName);
|
||||
if (!(await exists(keyPath))) {
|
||||
if (keyName === this.config.defaultSshKeyName && this.config.generateSshKeyIfMissing) {
|
||||
await this.initialize();
|
||||
}
|
||||
}
|
||||
|
||||
if (!(await exists(keyPath))) {
|
||||
throw new HttpError(404, "Public key not found");
|
||||
}
|
||||
|
||||
return (await readFile(keyPath, "utf8")).trim();
|
||||
}
|
||||
|
||||
getDefaultKeyName(): string {
|
||||
return this.config.defaultSshKeyName;
|
||||
}
|
||||
|
||||
private getPrivateKeyPath(name: string): string {
|
||||
return path.join(this.config.sshKeysDir, name);
|
||||
}
|
||||
|
||||
private getPublicKeyPath(name: string): string {
|
||||
return path.join(this.config.sshKeysDir, `${name}.pub`);
|
||||
}
|
||||
|
||||
private assertValidKeyName(name: string): void {
|
||||
if (!/^[a-zA-Z0-9._-]+$/.test(name)) {
|
||||
throw new HttpError(400, "Invalid ssh key filename");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function exists(filePath: string): Promise<boolean> {
|
||||
const fileStats = await stat(filePath).catch(() => null);
|
||||
return Boolean(fileStats?.isFile());
|
||||
}
|
||||
|
||||
async function runCommand(command: string, args: string[]): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(command, args);
|
||||
let stderr = "";
|
||||
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
reject(new Error(stderr.trim() || `${command} failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { RuntimeConfig } from "../config.js";
|
||||
import { HttpError } from "../errors.js";
|
||||
import { detectLanguage } from "../language.js";
|
||||
import { Semaphore } from "../lib/semaphore.js";
|
||||
import type { CountRequest, CountResult, LanguageStat } from "../types.js";
|
||||
import { KeyManager } from "./key-manager.js";
|
||||
|
||||
type CacheEntry = {
|
||||
expiresAt: number;
|
||||
value: Omit<CountResult, "cached">;
|
||||
};
|
||||
|
||||
export class RepoCounterService {
|
||||
private readonly cache = new Map<string, CacheEntry>();
|
||||
private readonly inFlight = new Map<string, Promise<Omit<CountResult, "cached">>>();
|
||||
private readonly semaphore: Semaphore;
|
||||
|
||||
constructor(
|
||||
private readonly config: RuntimeConfig,
|
||||
private readonly keyManager: KeyManager
|
||||
) {
|
||||
this.semaphore = new Semaphore(config.maxConcurrentScans);
|
||||
}
|
||||
|
||||
getHealth(): Record<string, number> {
|
||||
this.clearExpiredCache();
|
||||
|
||||
return {
|
||||
cacheEntries: this.cache.size,
|
||||
inFlight: this.inFlight.size,
|
||||
maxConcurrentScans: this.config.maxConcurrentScans,
|
||||
activeScans: this.semaphore.active,
|
||||
queuedScans: this.semaphore.queued
|
||||
};
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
await mkdir(this.config.tempRoot, { recursive: true });
|
||||
await this.cleanupStaleTempDirs();
|
||||
|
||||
setInterval(() => {
|
||||
this.clearExpiredCache();
|
||||
void this.cleanupStaleTempDirs();
|
||||
}, this.config.cacheSweepIntervalMs).unref();
|
||||
}
|
||||
|
||||
async count(request: CountRequest): Promise<CountResult> {
|
||||
const cacheKey = JSON.stringify(request);
|
||||
this.clearExpiredCache();
|
||||
|
||||
const cached = this.cache.get(cacheKey);
|
||||
if (cached && cached.expiresAt > Date.now()) {
|
||||
return { ...cached.value, cached: true };
|
||||
}
|
||||
|
||||
const active = this.inFlight.get(cacheKey);
|
||||
if (active) {
|
||||
const value = await active;
|
||||
return { ...value, cached: false };
|
||||
}
|
||||
|
||||
const task = this.semaphore.use(async () => {
|
||||
const value = await this.cloneAndCount(request);
|
||||
this.cache.set(cacheKey, {
|
||||
value,
|
||||
expiresAt: Date.now() + this.config.cacheTtlMs
|
||||
});
|
||||
return value;
|
||||
});
|
||||
|
||||
this.inFlight.set(cacheKey, task);
|
||||
|
||||
try {
|
||||
const value = await task;
|
||||
return { ...value, cached: false };
|
||||
} finally {
|
||||
this.inFlight.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
private async cloneAndCount(request: CountRequest): Promise<Omit<CountResult, "cached">> {
|
||||
const startedAt = Date.now();
|
||||
const repoDir = await mkdtemp(path.join(this.config.tempRoot, "repo-"));
|
||||
|
||||
try {
|
||||
await this.runGitClone(request.repo, repoDir, request.ref, request.sshKey);
|
||||
const stats = await countDirectory(repoDir);
|
||||
|
||||
return {
|
||||
repo: request.repo,
|
||||
ref: request.ref,
|
||||
sshKey: request.sshKey,
|
||||
lineCount: stats.lineCount,
|
||||
fileCount: stats.fileCount,
|
||||
languages: stats.languages,
|
||||
scannedAt: new Date().toISOString(),
|
||||
durationMs: Date.now() - startedAt
|
||||
};
|
||||
} finally {
|
||||
await rm(repoDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private async runGitClone(repo: string, targetDir: string, ref: string | null, sshKey: string | null): Promise<void> {
|
||||
const env = { ...process.env };
|
||||
const keyPath = await this.keyManager.resolvePrivateKeyPath(sshKey);
|
||||
|
||||
if (keyPath) {
|
||||
env.GIT_SSH_COMMAND = `ssh -i "${keyPath}" -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new`;
|
||||
}
|
||||
|
||||
await runCommand("git", ["clone", "--depth", "1", "--single-branch", repo, targetDir], env, this.config.cloneTimeoutMs);
|
||||
|
||||
if (ref) {
|
||||
await runCommand("git", ["-C", targetDir, "fetch", "--depth", "1", "origin", ref], env, this.config.cloneTimeoutMs);
|
||||
await runCommand("git", ["-C", targetDir, "checkout", "FETCH_HEAD"], env, this.config.cloneTimeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanupStaleTempDirs(): Promise<void> {
|
||||
const entries = await readdir(this.config.tempRoot, { withFileTypes: true }).catch(() => []);
|
||||
const staleBefore = Date.now() - this.config.cloneTimeoutMs * 2;
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || !entry.name.startsWith("repo-")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullPath = path.join(this.config.tempRoot, entry.name);
|
||||
const entryStats = await stat(fullPath).catch(() => null);
|
||||
if (!entryStats || entryStats.mtimeMs > staleBefore) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await rm(fullPath, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private clearExpiredCache(): void {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
if (entry.expiresAt <= now) {
|
||||
this.cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function countDirectory(rootDir: string): Promise<{
|
||||
fileCount: number;
|
||||
languages: LanguageStat[];
|
||||
lineCount: number;
|
||||
}> {
|
||||
let fileCount = 0;
|
||||
let lineCount = 0;
|
||||
const languages = new Map<string, LanguageStat>();
|
||||
const stack = [rootDir];
|
||||
|
||||
while (stack.length > 0) {
|
||||
const currentDir = stack.pop();
|
||||
if (!currentDir) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const entries = await readdir(currentDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.name === ".git") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullPath = path.join(currentDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
stack.push(fullPath);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.isFile() || !(await isTextFile(fullPath))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lines = await countFileLines(fullPath);
|
||||
const language = detectLanguage(fullPath);
|
||||
const current = languages.get(language) ?? { language, files: 0, lines: 0 };
|
||||
|
||||
current.files += 1;
|
||||
current.lines += lines;
|
||||
languages.set(language, current);
|
||||
|
||||
fileCount += 1;
|
||||
lineCount += lines;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
fileCount,
|
||||
lineCount,
|
||||
languages: Array.from(languages.values()).sort((a, b) => b.lines - a.lines || a.language.localeCompare(b.language))
|
||||
};
|
||||
}
|
||||
|
||||
async function isTextFile(filePath: string): Promise<boolean> {
|
||||
const buffer = await readFile(filePath);
|
||||
return !buffer.subarray(0, 4096).includes(0);
|
||||
}
|
||||
|
||||
async function countFileLines(filePath: string): Promise<number> {
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
let count = 0;
|
||||
let trailingChunk = "";
|
||||
const stream = createReadStream(filePath, { encoding: "utf8" });
|
||||
|
||||
stream.on("data", (chunk: string | Buffer) => {
|
||||
const text = trailingChunk + chunk.toString();
|
||||
const lines = text.split(/\r?\n/);
|
||||
trailingChunk = lines.pop() ?? "";
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.trim() !== "") {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
stream.on("end", () => {
|
||||
if (trailingChunk.trim() !== "") {
|
||||
count += 1;
|
||||
}
|
||||
resolve(count);
|
||||
});
|
||||
|
||||
stream.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function runCommand(
|
||||
command: string,
|
||||
args: string[],
|
||||
env: NodeJS.ProcessEnv,
|
||||
timeoutMs: number
|
||||
): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(command, args, { env });
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
}, timeoutMs);
|
||||
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
reject(error);
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
if (timedOut) {
|
||||
reject(new HttpError(504, `${command} timed out after ${Math.round(timeoutMs / 1000)}s`));
|
||||
return;
|
||||
}
|
||||
|
||||
reject(new HttpError(400, stderr.trim() || `${command} failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
export type LanguageStat = {
|
||||
language: string;
|
||||
files: number;
|
||||
lines: number;
|
||||
};
|
||||
|
||||
export type CountResult = {
|
||||
repo: string;
|
||||
ref: string | null;
|
||||
sshKey: string | null;
|
||||
cached: boolean;
|
||||
lineCount: number;
|
||||
fileCount: number;
|
||||
languages: LanguageStat[];
|
||||
scannedAt: string;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
export type CountRequest = {
|
||||
repo: string;
|
||||
ref: string | null;
|
||||
sshKey: string | null;
|
||||
};
|
||||
Reference in New Issue
Block a user