@@ -0,0 +1,5 @@
|
||||
node_modules
|
||||
dist
|
||||
.git
|
||||
.env
|
||||
tmp
|
||||
@@ -0,0 +1,12 @@
|
||||
root = true
|
||||
|
||||
[*.{js,ts,json,yml,yaml,md,sh}]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
@@ -0,0 +1,13 @@
|
||||
PORT=3000
|
||||
API_KEY=
|
||||
CACHE_TTL_MINUTES=5
|
||||
CACHE_SWEEP_INTERVAL_MINUTES=5
|
||||
RATE_LIMIT_WINDOW_MINUTES=5
|
||||
RATE_LIMIT_MAX=30
|
||||
MAX_CONCURRENT_SCANS=4
|
||||
CLONE_TIMEOUT_SECONDS=45
|
||||
DEFAULT_SSH_KEY_NAME=loc_via_git_ed25519
|
||||
GENERATE_SSH_KEY_IF_MISSING=false
|
||||
TRUST_PROXY=false
|
||||
SSH_KEYS_DIR=/app/keys
|
||||
TMP_DIR=/tmp/loc-via-git
|
||||
@@ -0,0 +1,29 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Test
|
||||
run: npm test
|
||||
@@ -0,0 +1,4 @@
|
||||
dist
|
||||
node_modules
|
||||
.env
|
||||
tmp
|
||||
@@ -0,0 +1,28 @@
|
||||
# Contributing
|
||||
|
||||
## Setup
|
||||
|
||||
1. Copy `.env.example` to `.env`.
|
||||
2. Install dependencies with `npm install`.
|
||||
3. Start the app with `npm run dev` or `docker compose up --build`.
|
||||
|
||||
## Before opening changes
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
npm test
|
||||
```
|
||||
|
||||
If you are touching the running service flow, also run:
|
||||
|
||||
```bash
|
||||
npm run smoke
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- Keep the code modular. Avoid adding unrelated logic into a single file.
|
||||
- Preserve the cleanup guarantees around temporary clone directories.
|
||||
- Do not commit private SSH keys.
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
FROM node:22-bookworm-slim AS build
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git openssh-client ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY package.json package-lock.json* tsconfig.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY src ./src
|
||||
RUN npm run build
|
||||
|
||||
FROM node:22-bookworm-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git openssh-client ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm ci --omit=dev
|
||||
|
||||
COPY --from=build /app/dist ./dist
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["npm", "start"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,27 @@
|
||||
PORT ?= 3000
|
||||
|
||||
.PHONY: install build dev start smoke docker-build docker-up docker-down
|
||||
|
||||
install:
|
||||
npm install
|
||||
|
||||
build:
|
||||
npm run build
|
||||
|
||||
dev:
|
||||
npm run dev
|
||||
|
||||
start:
|
||||
npm start
|
||||
|
||||
smoke:
|
||||
npm run smoke
|
||||
|
||||
docker-build:
|
||||
docker compose build
|
||||
|
||||
docker-up:
|
||||
docker compose up -d
|
||||
|
||||
docker-down:
|
||||
docker compose down
|
||||
@@ -0,0 +1,110 @@
|
||||
# loc-via-git
|
||||
|
||||
Tiny API that clones a Git repo and counts its non-empty lines of code.
|
||||
|
||||
## Features
|
||||
|
||||
- Plain text and JSON endpoints
|
||||
- Optional API key enforcement
|
||||
- Per-repo in-memory caching
|
||||
- Extension-based language breakdown
|
||||
- Basic rate limiting
|
||||
- Bounded concurrent scans so the host does not get hammered
|
||||
- SSH key file support for private repos
|
||||
- Docker Compose deployment
|
||||
|
||||
## Endpoints
|
||||
|
||||
- `GET /loc.txt?repo=<git-url>&ssh_key=<optional-key-file>&ref=<optional-ref>`
|
||||
- Returns the line count as plain text.
|
||||
- `GET /loc?repo=<git-url>&ssh_key=<optional-key-file>&ref=<optional-ref>`
|
||||
- Returns JSON metadata, including a language breakdown by files and non-empty lines.
|
||||
- `GET /health`
|
||||
- Health plus queue/cache stats.
|
||||
|
||||
## Auth
|
||||
|
||||
No auth by default. If `API_KEY` is set, send it as either:
|
||||
|
||||
- `x-api-key: ...`
|
||||
- `Authorization: Bearer ...`
|
||||
|
||||
## SSH keys
|
||||
|
||||
SSH keys live inside the Docker volume mounted at `/app/keys`, not in the repo or a host bind mount.
|
||||
|
||||
You have two options:
|
||||
|
||||
- Provide a key pair yourself inside the `ssh_keys` Docker volume
|
||||
- Let the service generate the default key by setting `GENERATE_SSH_KEY_IF_MISSING=true`
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
curl "http://localhost:3000/loc?repo=ssh://git@example.com/org/repo.git&ssh_key=loc_via_git_ed25519"
|
||||
```
|
||||
|
||||
Fetch the public key to add it on the Git host:
|
||||
|
||||
```bash
|
||||
curl http://localhost:3000/ssh/public-key
|
||||
```
|
||||
|
||||
You can also request a specific key:
|
||||
|
||||
```bash
|
||||
curl "http://localhost:3000/ssh/public-key?ssh_key=loc_via_git_ed25519"
|
||||
```
|
||||
|
||||
## Cleanup and caching
|
||||
|
||||
- Every clone happens in a temporary directory and is deleted in a `finally` block after the scan finishes or fails.
|
||||
- The service also sweeps stale temp directories in case a process dies mid-scan.
|
||||
- Cache entries live in memory only and expire after `CACHE_TTL_MINUTES`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Copy `.env.example` to `.env` and adjust:
|
||||
|
||||
```env
|
||||
PORT=3000
|
||||
API_KEY=
|
||||
CACHE_TTL_MINUTES=5
|
||||
CACHE_SWEEP_INTERVAL_MINUTES=5
|
||||
RATE_LIMIT_WINDOW_MINUTES=5
|
||||
RATE_LIMIT_MAX=30
|
||||
MAX_CONCURRENT_SCANS=4
|
||||
CLONE_TIMEOUT_SECONDS=45
|
||||
DEFAULT_SSH_KEY_NAME=loc_via_git_ed25519
|
||||
GENERATE_SSH_KEY_IF_MISSING=false
|
||||
TRUST_PROXY=false
|
||||
SSH_KEYS_DIR=/app/keys
|
||||
TMP_DIR=/tmp/loc-via-git
|
||||
```
|
||||
|
||||
## Local run
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
npm start
|
||||
```
|
||||
|
||||
## Docker compose
|
||||
|
||||
```bash
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
To copy an existing key pair into the Docker volume:
|
||||
|
||||
```bash
|
||||
docker cp ./id_ed25519 loc-via-git-api-1:/app/keys/loc_via_git_ed25519
|
||||
docker cp ./id_ed25519.pub loc-via-git-api-1:/app/keys/loc_via_git_ed25519.pub
|
||||
```
|
||||
|
||||
## Smoke test
|
||||
|
||||
```bash
|
||||
npm run smoke
|
||||
```
|
||||
@@ -0,0 +1,35 @@
|
||||
services:
|
||||
api:
|
||||
build: .
|
||||
env_file:
|
||||
- .env
|
||||
healthcheck:
|
||||
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/health').then((r) => { if (!r.ok) process.exit(1); }).catch(() => process.exit(1))"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
ports:
|
||||
- "${PORT:-3000}:3000"
|
||||
environment:
|
||||
PORT: 3000
|
||||
API_KEY: ${API_KEY:-}
|
||||
CACHE_TTL_MINUTES: ${CACHE_TTL_MINUTES:-5}
|
||||
CACHE_SWEEP_INTERVAL_MINUTES: ${CACHE_SWEEP_INTERVAL_MINUTES:-5}
|
||||
RATE_LIMIT_WINDOW_MINUTES: ${RATE_LIMIT_WINDOW_MINUTES:-5}
|
||||
RATE_LIMIT_MAX: ${RATE_LIMIT_MAX:-30}
|
||||
MAX_CONCURRENT_SCANS: ${MAX_CONCURRENT_SCANS:-4}
|
||||
CLONE_TIMEOUT_SECONDS: ${CLONE_TIMEOUT_SECONDS:-45}
|
||||
DEFAULT_SSH_KEY_NAME: ${DEFAULT_SSH_KEY_NAME:-loc_via_git_ed25519}
|
||||
GENERATE_SSH_KEY_IF_MISSING: ${GENERATE_SSH_KEY_IF_MISSING:-false}
|
||||
TRUST_PROXY: ${TRUST_PROXY:-false}
|
||||
SSH_KEYS_DIR: /app/keys
|
||||
TMP_DIR: /tmp/loc-via-git
|
||||
volumes:
|
||||
- ssh_keys:/app/keys
|
||||
tmpfs:
|
||||
- /tmp/loc-via-git
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
ssh_keys:
|
||||
Generated
+1751
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "loc-via-git",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"smoke": "bash tests/smoke.sh",
|
||||
"start": "node dist/server.js",
|
||||
"test": "npm run build && node --test 'tests/**/*.test.js'"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^4.21.2",
|
||||
"express-rate-limit": "^7.4.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/express": "^5.0.3",
|
||||
"@types/node": "^24.0.15",
|
||||
"supertest": "^7.1.1",
|
||||
"tsx": "^4.20.3",
|
||||
"typescript": "^5.9.2"
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
import express, { type NextFunction, type Request, type Response } from "express";
|
||||
import rateLimit from "express-rate-limit";
|
||||
|
||||
import type { RuntimeConfig } from "./config.js";
|
||||
import { HttpError } from "./errors.js";
|
||||
import { asyncHandler, readOptionalString } from "./lib/http.js";
|
||||
import type { CountRequest } from "./types.js";
|
||||
|
||||
type AppDependencies = {
|
||||
getHealth: () => Record<string, number>;
|
||||
getPublicKey: (name: string | null) => Promise<string>;
|
||||
getDefaultKeyName: () => string;
|
||||
count: (request: CountRequest) => Promise<import("./types.js").CountResult>;
|
||||
};
|
||||
|
||||
export function createApp(config: RuntimeConfig, deps: AppDependencies) {
|
||||
const app = express();
|
||||
app.set("trust proxy", config.trustProxy);
|
||||
|
||||
app.use(rateLimit({
|
||||
windowMs: config.rateLimitWindowMs,
|
||||
limit: config.rateLimitMax,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false
|
||||
}));
|
||||
|
||||
app.get("/health", (_req, res) => {
|
||||
res.json({ ok: true, ...deps.getHealth() });
|
||||
});
|
||||
|
||||
app.get("/ssh/public-key", asyncHandler(async (req, res) => {
|
||||
const keyName = readOptionalString(req.query.ssh_key) ?? deps.getDefaultKeyName();
|
||||
const publicKey = await deps.getPublicKey(keyName);
|
||||
res.type("text/plain").send(publicKey);
|
||||
}));
|
||||
|
||||
app.use((req, res, next) => {
|
||||
if (!config.apiKey || req.path === "/ssh/public-key") {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
const headerKey = req.header("x-api-key");
|
||||
const bearer = req.header("authorization")?.replace(/^Bearer\s+/i, "").trim();
|
||||
|
||||
if (headerKey === config.apiKey || bearer === config.apiKey) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
res.status(401).json({ error: "Unauthorized" });
|
||||
});
|
||||
|
||||
app.get("/loc.txt", asyncHandler(async (req, res) => {
|
||||
const result = await deps.count(readCountRequest(req));
|
||||
res.type("text/plain").send(String(result.lineCount));
|
||||
}));
|
||||
|
||||
app.get("/loc", asyncHandler(async (req, res) => {
|
||||
const result = await deps.count(readCountRequest(req));
|
||||
res.json(result);
|
||||
}));
|
||||
|
||||
app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
const status = error instanceof HttpError ? error.statusCode : message.startsWith("Missing ") ? 400 : 500;
|
||||
res.status(status).json({ error: message });
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
function readCountRequest(req: Request): CountRequest {
|
||||
const repo = typeof req.query.repo === "string" ? req.query.repo.trim() : "";
|
||||
if (!repo) {
|
||||
throw new HttpError(400, "Missing repo query parameter");
|
||||
}
|
||||
|
||||
return {
|
||||
repo,
|
||||
ref: readOptionalString(req.query.ref),
|
||||
sshKey: readOptionalString(req.query.ssh_key)
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
export type RuntimeConfig = {
|
||||
apiKey: string;
|
||||
cacheSweepIntervalMs: number;
|
||||
cacheTtlMs: number;
|
||||
cloneTimeoutMs: number;
|
||||
defaultSshKeyName: string;
|
||||
generateSshKeyIfMissing: boolean;
|
||||
maxConcurrentScans: number;
|
||||
port: number;
|
||||
rateLimitMax: number;
|
||||
rateLimitWindowMs: number;
|
||||
sshKeysDir: string;
|
||||
tempRoot: string;
|
||||
trustProxy: boolean | number | string;
|
||||
};
|
||||
|
||||
export function loadConfig(): RuntimeConfig {
|
||||
return {
|
||||
apiKey: readEnvString("API_KEY", ""),
|
||||
cacheSweepIntervalMs: readEnvMinutes("CACHE_SWEEP_INTERVAL_MINUTES", 5),
|
||||
cacheTtlMs: readEnvMinutes("CACHE_TTL_MINUTES", 5),
|
||||
cloneTimeoutMs: readEnvSeconds("CLONE_TIMEOUT_SECONDS", 45),
|
||||
defaultSshKeyName: readEnvString("DEFAULT_SSH_KEY_NAME", "loc_via_git_ed25519"),
|
||||
generateSshKeyIfMissing: readEnvBoolean("GENERATE_SSH_KEY_IF_MISSING", false),
|
||||
maxConcurrentScans: readEnvNumber("MAX_CONCURRENT_SCANS", 4),
|
||||
port: readEnvNumber("PORT", 3000),
|
||||
rateLimitMax: readEnvNumber("RATE_LIMIT_MAX", 30),
|
||||
rateLimitWindowMs: readEnvMinutes("RATE_LIMIT_WINDOW_MINUTES", 5),
|
||||
sshKeysDir: readEnvString("SSH_KEYS_DIR", path.resolve(process.cwd(), "keys")),
|
||||
tempRoot: readEnvString("TMP_DIR", path.join(tmpdir(), "loc-via-git")),
|
||||
trustProxy: readEnvTrustProxy("TRUST_PROXY", false)
|
||||
};
|
||||
}
|
||||
|
||||
function readEnvMinutes(name: string, fallback: number): number {
|
||||
return readEnvNumber(name, fallback) * 60_000;
|
||||
}
|
||||
|
||||
function readEnvSeconds(name: string, fallback: number): number {
|
||||
return readEnvNumber(name, fallback) * 1000;
|
||||
}
|
||||
|
||||
function readEnvNumber(name: string, fallback: number): number {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
throw new Error(`Invalid ${name}: ${raw}`);
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readEnvBoolean(name: string, fallback: boolean): boolean {
|
||||
const raw = process.env[name]?.trim().toLowerCase();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
if (["1", "true", "yes", "on"].includes(raw)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (["0", "false", "no", "off"].includes(raw)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
throw new Error(`Invalid ${name}: ${raw}`);
|
||||
}
|
||||
|
||||
function readEnvTrustProxy(name: string, fallback: boolean | number | string): boolean | number | string {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
if (["true", "false", "1", "0", "yes", "no", "on", "off"].includes(raw.toLowerCase())) {
|
||||
return readEnvBoolean(name, Boolean(fallback));
|
||||
}
|
||||
|
||||
const asNumber = Number(raw);
|
||||
if (Number.isInteger(asNumber) && asNumber >= 0) {
|
||||
return asNumber;
|
||||
}
|
||||
|
||||
return raw;
|
||||
}
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
return process.env[name]?.trim() ?? fallback;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
public readonly statusCode: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import path from "node:path";
|
||||
|
||||
const extensionToLanguage: Record<string, string> = {
|
||||
".c": "C",
|
||||
".cc": "C++",
|
||||
".cpp": "C++",
|
||||
".cs": "C#",
|
||||
".css": "CSS",
|
||||
".go": "Go",
|
||||
".h": "C/C++ Header",
|
||||
".hpp": "C++ Header",
|
||||
".html": "HTML",
|
||||
".java": "Java",
|
||||
".js": "JavaScript",
|
||||
".json": "JSON",
|
||||
".jsx": "JavaScript React",
|
||||
".kt": "Kotlin",
|
||||
".lua": "Lua",
|
||||
".md": "Markdown",
|
||||
".mjs": "JavaScript",
|
||||
".php": "PHP",
|
||||
".py": "Python",
|
||||
".rb": "Ruby",
|
||||
".rs": "Rust",
|
||||
".scss": "SCSS",
|
||||
".sh": "Shell",
|
||||
".sql": "SQL",
|
||||
".svg": "SVG",
|
||||
".svelte": "Svelte",
|
||||
".swift": "Swift",
|
||||
".toml": "TOML",
|
||||
".ts": "TypeScript",
|
||||
".tsx": "TypeScript React",
|
||||
".txt": "Plain Text",
|
||||
".vue": "Vue",
|
||||
".xml": "XML",
|
||||
".yaml": "YAML",
|
||||
".yml": "YAML"
|
||||
};
|
||||
|
||||
export function detectLanguage(filePath: string): string {
|
||||
const fileName = path.basename(filePath).toLowerCase();
|
||||
const extension = path.extname(fileName);
|
||||
|
||||
if (fileName === "dockerfile") {
|
||||
return "Dockerfile";
|
||||
}
|
||||
|
||||
if (fileName.endsWith(".d.ts")) {
|
||||
return "TypeScript";
|
||||
}
|
||||
|
||||
return extensionToLanguage[extension] ?? "Plain Text";
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
|
||||
export function asyncHandler(handler: (req: Request, res: Response) => Promise<void>) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
void handler(req, res).catch(next);
|
||||
};
|
||||
}
|
||||
|
||||
export function readOptionalString(value: unknown): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trimmed = value.trim();
|
||||
return trimmed === "" ? null : trimmed;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
export class Semaphore {
|
||||
private current = 0;
|
||||
private readonly waiting: Array<() => void> = [];
|
||||
|
||||
constructor(private readonly limit: number) {}
|
||||
|
||||
get active(): number {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
get queued(): number {
|
||||
return this.waiting.length;
|
||||
}
|
||||
|
||||
async use<T>(task: () => Promise<T>): Promise<T> {
|
||||
await this.acquire();
|
||||
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
this.release();
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<void> {
|
||||
if (this.current < this.limit) {
|
||||
this.current += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
this.waiting.push(() => {
|
||||
this.current += 1;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private release(): void {
|
||||
this.current -= 1;
|
||||
const next = this.waiting.shift();
|
||||
next?.();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { loadConfig } from "./config.js";
|
||||
import { createApp } from "./app.js";
|
||||
import { KeyManager } from "./services/key-manager.js";
|
||||
import { RepoCounterService } from "./services/repo-counter.js";
|
||||
|
||||
const config = loadConfig();
|
||||
const keyManager = new KeyManager(config);
|
||||
const repoCounter = new RepoCounterService(config, keyManager);
|
||||
|
||||
await keyManager.initialize();
|
||||
await repoCounter.initialize();
|
||||
|
||||
const app = createApp(config, {
|
||||
getHealth: () => repoCounter.getHealth(),
|
||||
getPublicKey: (name) => keyManager.getPublicKey(name),
|
||||
getDefaultKeyName: () => keyManager.getDefaultKeyName(),
|
||||
count: (request) => repoCounter.count(request)
|
||||
});
|
||||
|
||||
app.listen(config.port, () => {
|
||||
console.log(`loc-via-git listening on ${config.port}`);
|
||||
});
|
||||
@@ -0,0 +1,118 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir, readFile, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { RuntimeConfig } from "../config.js";
|
||||
import { HttpError } from "../errors.js";
|
||||
|
||||
export class KeyManager {
|
||||
constructor(private readonly config: RuntimeConfig) {}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
await mkdir(this.config.sshKeysDir, { recursive: true });
|
||||
|
||||
if (!this.config.generateSshKeyIfMissing) {
|
||||
return;
|
||||
}
|
||||
|
||||
const privateKeyPath = this.getPrivateKeyPath(this.config.defaultSshKeyName);
|
||||
const publicKeyPath = this.getPublicKeyPath(this.config.defaultSshKeyName);
|
||||
const privateExists = await exists(privateKeyPath);
|
||||
const publicExists = await exists(publicKeyPath);
|
||||
|
||||
if (privateExists && publicExists) {
|
||||
return;
|
||||
}
|
||||
|
||||
await runCommand("ssh-keygen", [
|
||||
"-t", "ed25519",
|
||||
"-N", "",
|
||||
"-f", privateKeyPath,
|
||||
"-C", "loc-via-git"
|
||||
]);
|
||||
}
|
||||
|
||||
async resolvePrivateKeyPath(name: string | null): Promise<string | null> {
|
||||
const keyName = name ?? this.config.defaultSshKeyName;
|
||||
if (!keyName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.assertValidKeyName(keyName);
|
||||
|
||||
const keyPath = this.getPrivateKeyPath(keyName);
|
||||
if (!(await exists(keyPath))) {
|
||||
if (keyName === this.config.defaultSshKeyName && this.config.generateSshKeyIfMissing) {
|
||||
await this.initialize();
|
||||
}
|
||||
}
|
||||
|
||||
if (!(await exists(keyPath))) {
|
||||
throw new HttpError(400, "SSH key not found");
|
||||
}
|
||||
|
||||
return keyPath;
|
||||
}
|
||||
|
||||
async getPublicKey(name: string | null): Promise<string> {
|
||||
const keyName = name ?? this.config.defaultSshKeyName;
|
||||
this.assertValidKeyName(keyName);
|
||||
|
||||
const keyPath = this.getPublicKeyPath(keyName);
|
||||
if (!(await exists(keyPath))) {
|
||||
if (keyName === this.config.defaultSshKeyName && this.config.generateSshKeyIfMissing) {
|
||||
await this.initialize();
|
||||
}
|
||||
}
|
||||
|
||||
if (!(await exists(keyPath))) {
|
||||
throw new HttpError(404, "Public key not found");
|
||||
}
|
||||
|
||||
return (await readFile(keyPath, "utf8")).trim();
|
||||
}
|
||||
|
||||
getDefaultKeyName(): string {
|
||||
return this.config.defaultSshKeyName;
|
||||
}
|
||||
|
||||
private getPrivateKeyPath(name: string): string {
|
||||
return path.join(this.config.sshKeysDir, name);
|
||||
}
|
||||
|
||||
private getPublicKeyPath(name: string): string {
|
||||
return path.join(this.config.sshKeysDir, `${name}.pub`);
|
||||
}
|
||||
|
||||
private assertValidKeyName(name: string): void {
|
||||
if (!/^[a-zA-Z0-9._-]+$/.test(name)) {
|
||||
throw new HttpError(400, "Invalid ssh key filename");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function exists(filePath: string): Promise<boolean> {
|
||||
const fileStats = await stat(filePath).catch(() => null);
|
||||
return Boolean(fileStats?.isFile());
|
||||
}
|
||||
|
||||
async function runCommand(command: string, args: string[]): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(command, args);
|
||||
let stderr = "";
|
||||
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
reject(new Error(stderr.trim() || `${command} failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { RuntimeConfig } from "../config.js";
|
||||
import { HttpError } from "../errors.js";
|
||||
import { detectLanguage } from "../language.js";
|
||||
import { Semaphore } from "../lib/semaphore.js";
|
||||
import type { CountRequest, CountResult, LanguageStat } from "../types.js";
|
||||
import { KeyManager } from "./key-manager.js";
|
||||
|
||||
type CacheEntry = {
|
||||
expiresAt: number;
|
||||
value: Omit<CountResult, "cached">;
|
||||
};
|
||||
|
||||
export class RepoCounterService {
|
||||
private readonly cache = new Map<string, CacheEntry>();
|
||||
private readonly inFlight = new Map<string, Promise<Omit<CountResult, "cached">>>();
|
||||
private readonly semaphore: Semaphore;
|
||||
|
||||
constructor(
|
||||
private readonly config: RuntimeConfig,
|
||||
private readonly keyManager: KeyManager
|
||||
) {
|
||||
this.semaphore = new Semaphore(config.maxConcurrentScans);
|
||||
}
|
||||
|
||||
getHealth(): Record<string, number> {
|
||||
this.clearExpiredCache();
|
||||
|
||||
return {
|
||||
cacheEntries: this.cache.size,
|
||||
inFlight: this.inFlight.size,
|
||||
maxConcurrentScans: this.config.maxConcurrentScans,
|
||||
activeScans: this.semaphore.active,
|
||||
queuedScans: this.semaphore.queued
|
||||
};
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
await mkdir(this.config.tempRoot, { recursive: true });
|
||||
await this.cleanupStaleTempDirs();
|
||||
|
||||
setInterval(() => {
|
||||
this.clearExpiredCache();
|
||||
void this.cleanupStaleTempDirs();
|
||||
}, this.config.cacheSweepIntervalMs).unref();
|
||||
}
|
||||
|
||||
async count(request: CountRequest): Promise<CountResult> {
|
||||
const cacheKey = JSON.stringify(request);
|
||||
this.clearExpiredCache();
|
||||
|
||||
const cached = this.cache.get(cacheKey);
|
||||
if (cached && cached.expiresAt > Date.now()) {
|
||||
return { ...cached.value, cached: true };
|
||||
}
|
||||
|
||||
const active = this.inFlight.get(cacheKey);
|
||||
if (active) {
|
||||
const value = await active;
|
||||
return { ...value, cached: false };
|
||||
}
|
||||
|
||||
const task = this.semaphore.use(async () => {
|
||||
const value = await this.cloneAndCount(request);
|
||||
this.cache.set(cacheKey, {
|
||||
value,
|
||||
expiresAt: Date.now() + this.config.cacheTtlMs
|
||||
});
|
||||
return value;
|
||||
});
|
||||
|
||||
this.inFlight.set(cacheKey, task);
|
||||
|
||||
try {
|
||||
const value = await task;
|
||||
return { ...value, cached: false };
|
||||
} finally {
|
||||
this.inFlight.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
private async cloneAndCount(request: CountRequest): Promise<Omit<CountResult, "cached">> {
|
||||
const startedAt = Date.now();
|
||||
const repoDir = await mkdtemp(path.join(this.config.tempRoot, "repo-"));
|
||||
|
||||
try {
|
||||
await this.runGitClone(request.repo, repoDir, request.ref, request.sshKey);
|
||||
const stats = await countDirectory(repoDir);
|
||||
|
||||
return {
|
||||
repo: request.repo,
|
||||
ref: request.ref,
|
||||
sshKey: request.sshKey,
|
||||
lineCount: stats.lineCount,
|
||||
fileCount: stats.fileCount,
|
||||
languages: stats.languages,
|
||||
scannedAt: new Date().toISOString(),
|
||||
durationMs: Date.now() - startedAt
|
||||
};
|
||||
} finally {
|
||||
await rm(repoDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private async runGitClone(repo: string, targetDir: string, ref: string | null, sshKey: string | null): Promise<void> {
|
||||
const env = { ...process.env };
|
||||
const keyPath = await this.keyManager.resolvePrivateKeyPath(sshKey);
|
||||
|
||||
if (keyPath) {
|
||||
env.GIT_SSH_COMMAND = `ssh -i "${keyPath}" -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new`;
|
||||
}
|
||||
|
||||
await runCommand("git", ["clone", "--depth", "1", "--single-branch", repo, targetDir], env, this.config.cloneTimeoutMs);
|
||||
|
||||
if (ref) {
|
||||
await runCommand("git", ["-C", targetDir, "fetch", "--depth", "1", "origin", ref], env, this.config.cloneTimeoutMs);
|
||||
await runCommand("git", ["-C", targetDir, "checkout", "FETCH_HEAD"], env, this.config.cloneTimeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanupStaleTempDirs(): Promise<void> {
|
||||
const entries = await readdir(this.config.tempRoot, { withFileTypes: true }).catch(() => []);
|
||||
const staleBefore = Date.now() - this.config.cloneTimeoutMs * 2;
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || !entry.name.startsWith("repo-")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullPath = path.join(this.config.tempRoot, entry.name);
|
||||
const entryStats = await stat(fullPath).catch(() => null);
|
||||
if (!entryStats || entryStats.mtimeMs > staleBefore) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await rm(fullPath, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private clearExpiredCache(): void {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
if (entry.expiresAt <= now) {
|
||||
this.cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function countDirectory(rootDir: string): Promise<{
|
||||
fileCount: number;
|
||||
languages: LanguageStat[];
|
||||
lineCount: number;
|
||||
}> {
|
||||
let fileCount = 0;
|
||||
let lineCount = 0;
|
||||
const languages = new Map<string, LanguageStat>();
|
||||
const stack = [rootDir];
|
||||
|
||||
while (stack.length > 0) {
|
||||
const currentDir = stack.pop();
|
||||
if (!currentDir) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const entries = await readdir(currentDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.name === ".git") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullPath = path.join(currentDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
stack.push(fullPath);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.isFile() || !(await isTextFile(fullPath))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lines = await countFileLines(fullPath);
|
||||
const language = detectLanguage(fullPath);
|
||||
const current = languages.get(language) ?? { language, files: 0, lines: 0 };
|
||||
|
||||
current.files += 1;
|
||||
current.lines += lines;
|
||||
languages.set(language, current);
|
||||
|
||||
fileCount += 1;
|
||||
lineCount += lines;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
fileCount,
|
||||
lineCount,
|
||||
languages: Array.from(languages.values()).sort((a, b) => b.lines - a.lines || a.language.localeCompare(b.language))
|
||||
};
|
||||
}
|
||||
|
||||
async function isTextFile(filePath: string): Promise<boolean> {
|
||||
const buffer = await readFile(filePath);
|
||||
return !buffer.subarray(0, 4096).includes(0);
|
||||
}
|
||||
|
||||
async function countFileLines(filePath: string): Promise<number> {
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
let count = 0;
|
||||
let trailingChunk = "";
|
||||
const stream = createReadStream(filePath, { encoding: "utf8" });
|
||||
|
||||
stream.on("data", (chunk: string | Buffer) => {
|
||||
const text = trailingChunk + chunk.toString();
|
||||
const lines = text.split(/\r?\n/);
|
||||
trailingChunk = lines.pop() ?? "";
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.trim() !== "") {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
stream.on("end", () => {
|
||||
if (trailingChunk.trim() !== "") {
|
||||
count += 1;
|
||||
}
|
||||
resolve(count);
|
||||
});
|
||||
|
||||
stream.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function runCommand(
|
||||
command: string,
|
||||
args: string[],
|
||||
env: NodeJS.ProcessEnv,
|
||||
timeoutMs: number
|
||||
): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(command, args, { env });
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
}, timeoutMs);
|
||||
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
reject(error);
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
if (timedOut) {
|
||||
reject(new HttpError(504, `${command} timed out after ${Math.round(timeoutMs / 1000)}s`));
|
||||
return;
|
||||
}
|
||||
|
||||
reject(new HttpError(400, stderr.trim() || `${command} failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
export type LanguageStat = {
|
||||
language: string;
|
||||
files: number;
|
||||
lines: number;
|
||||
};
|
||||
|
||||
export type CountResult = {
|
||||
repo: string;
|
||||
ref: string | null;
|
||||
sshKey: string | null;
|
||||
cached: boolean;
|
||||
lineCount: number;
|
||||
fileCount: number;
|
||||
languages: LanguageStat[];
|
||||
scannedAt: string;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
export type CountRequest = {
|
||||
repo: string;
|
||||
ref: string | null;
|
||||
sshKey: string | null;
|
||||
};
|
||||
@@ -0,0 +1,77 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import request from "supertest";
|
||||
|
||||
import { createApp } from "../dist/app.js";
|
||||
|
||||
const baseConfig = {
|
||||
apiKey: "",
|
||||
cacheSweepIntervalMs: 300000,
|
||||
cacheTtlMs: 300000,
|
||||
cloneTimeoutMs: 45000,
|
||||
defaultSshKeyName: "loc_via_git_ed25519",
|
||||
generateSshKeyIfMissing: false,
|
||||
maxConcurrentScans: 4,
|
||||
port: 3000,
|
||||
rateLimitMax: 30,
|
||||
rateLimitWindowMs: 300000,
|
||||
sshKeysDir: "/tmp/keys",
|
||||
tempRoot: "/tmp/loc-via-git",
|
||||
trustProxy: false
|
||||
};
|
||||
|
||||
test("GET /loc returns count metadata", async () => {
|
||||
const app = createApp(baseConfig, {
|
||||
getHealth: () => ({ cacheEntries: 1, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }),
|
||||
getPublicKey: async () => "ssh-ed25519 AAAA",
|
||||
getDefaultKeyName: () => "loc_via_git_ed25519",
|
||||
count: async (requestInput) => ({
|
||||
repo: requestInput.repo,
|
||||
ref: requestInput.ref,
|
||||
sshKey: requestInput.sshKey,
|
||||
cached: false,
|
||||
lineCount: 12,
|
||||
fileCount: 3,
|
||||
languages: [{ language: "TypeScript", files: 3, lines: 12 }],
|
||||
scannedAt: "2026-01-01T00:00:00.000Z",
|
||||
durationMs: 123
|
||||
})
|
||||
});
|
||||
|
||||
const response = await request(app).get("/loc?repo=https://example.com/repo.git");
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.body.lineCount, 12);
|
||||
assert.equal(response.body.languages[0].language, "TypeScript");
|
||||
});
|
||||
|
||||
test("GET /ssh/public-key returns text without auth", async () => {
|
||||
const app = createApp({ ...baseConfig, apiKey: "secret" }, {
|
||||
getHealth: () => ({ cacheEntries: 0, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }),
|
||||
getPublicKey: async () => "ssh-ed25519 AAAAB3 key",
|
||||
getDefaultKeyName: () => "loc_via_git_ed25519",
|
||||
count: async () => {
|
||||
throw new Error("not used");
|
||||
}
|
||||
});
|
||||
|
||||
const response = await request(app).get("/ssh/public-key");
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.text, "ssh-ed25519 AAAAB3 key");
|
||||
});
|
||||
|
||||
test("GET /loc enforces api key when configured", async () => {
|
||||
const app = createApp({ ...baseConfig, apiKey: "secret" }, {
|
||||
getHealth: () => ({ cacheEntries: 0, inFlight: 0, maxConcurrentScans: 4, activeScans: 0, queuedScans: 0 }),
|
||||
getPublicKey: async () => "ssh-ed25519 AAAA",
|
||||
getDefaultKeyName: () => "loc_via_git_ed25519",
|
||||
count: async () => {
|
||||
throw new Error("not used");
|
||||
}
|
||||
});
|
||||
|
||||
const response = await request(app).get("/loc?repo=https://example.com/repo.git");
|
||||
|
||||
assert.equal(response.status, 401);
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { detectLanguage } from "../dist/language.js";
|
||||
|
||||
test("detectLanguage recognizes common source files", () => {
|
||||
assert.equal(detectLanguage("/tmp/example.tsx"), "TypeScript React");
|
||||
assert.equal(detectLanguage("/tmp/Dockerfile"), "Dockerfile");
|
||||
assert.equal(detectLanguage("/tmp/types.d.ts"), "TypeScript");
|
||||
assert.equal(detectLanguage("/tmp/file.unknown"), "Plain Text");
|
||||
});
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
base_url="${1:-http://127.0.0.1:3000}"
|
||||
repo="${2:-https://github.com/octocat/Hello-World.git}"
|
||||
|
||||
health="$(curl -fsS "$base_url/health")"
|
||||
loc_json="$(curl -fsS "$base_url/loc?repo=$repo")"
|
||||
loc_text="$(curl -fsS "$base_url/loc.txt?repo=$repo")"
|
||||
|
||||
printf 'health: %s\n' "$health"
|
||||
printf 'loc json: %s\n' "$loc_json"
|
||||
printf 'loc text: %s\n' "$loc_text"
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user