Compare commits

...

5 Commits

Author SHA1 Message Date
Space-Banane c94bc7e5d4 Surface programming errors in user fetches
ci / push-image (push) Successful in 1m14s
ci / deploy-coolify (push) Successful in 7s
ci / test (push) Successful in 36s
2026-06-04 22:56:11 +02:00
Space-Banane c622420e38 Harden multi-user activity fetching
ci / deploy-coolify (push) Successful in 7s
ci / test (push) Successful in 13s
ci / push-image (push) Successful in 55s
2026-06-04 22:51:48 +02:00
Space-Banane 4444174365 Add multi-username activity support
ci / test (push) Successful in 10s
ci / push-image (push) Successful in 52s
ci / deploy-coolify (push) Successful in 6s
2026-06-04 22:42:48 +02:00
Space-Banane ad792b8137 Fix cross-year GitHub activity ranges
ci / test (push) Successful in 24s
ci / push-image (push) Successful in 1m8s
ci / deploy-coolify (push) Successful in 7s
2026-06-03 13:41:04 +02:00
Space-Banane 490f053c64 Improve activity response metadata 2026-06-03 13:41:04 +02:00
8 changed files with 458 additions and 39 deletions
+11 -2
View File
@@ -36,6 +36,8 @@ Supported query params:
- `theme=dark|light`
- `source=all|github|gitea`
Responses also include an `X-Activity-Stale: true|false` header so image consumers can detect when stale cached data was used after an upstream fetch failure.
Examples:
```bash
@@ -60,9 +62,16 @@ Copy `.env.example` to `.env` and edit values.
Required:
- `GITHUB_USERNAME`
- `GITHUB_USERNAME` (supports a comma-separated list, for example `octocat,hubot`)
- `GITEA_BASE_URL`
- `GITEA_USERNAME`
- `GITEA_USERNAME` (supports a comma-separated list)
Example:
```env
GITHUB_USERNAME=octocat,hubot
GITEA_USERNAME=alice,bob
```
Optional:
+86 -14
View File
@@ -1,3 +1,4 @@
import asyncio
import hashlib
import json
import logging
@@ -113,6 +114,58 @@ async def _fetch_with_cache(
raise
async def _fetch_source_usernames(
cache: FileCache | None,
source_prefix: str,
usernames: list[str],
from_date: date,
to_date: date,
fetcher_factory,
) -> tuple[dict[str, int], bool]:
if not usernames:
return {}, False
fetch_tasks: list[asyncio.Task[tuple[str, tuple[dict[str, int], bool] | Exception]]] = []
for username in usernames:
key = f"{source_prefix}_{username}_{from_date}_{to_date}"
fetch_tasks.append(
asyncio.create_task(
_fetch_source_username(cache, key, username, fetcher_factory(username))
)
)
combined: dict[str, int] = {}
stale = False
failures: list[Exception] = []
for username, result in await asyncio.gather(*fetch_tasks):
if isinstance(result, Exception):
logger.warning("source fetch failed for %s/%s, skipping user: %s", source_prefix, username, result)
failures.append(result)
continue
data, entry_stale = result
stale = stale or entry_stale
for day, count in data.items():
combined[day] = combined.get(day, 0) + int(count)
if not combined and failures:
raise failures[0]
return combined, stale
async def _fetch_source_username(
cache: FileCache | None,
key: str,
username: str,
fetcher,
) -> tuple[str, tuple[dict[str, int], bool] | Exception]:
try:
return username, await _fetch_with_cache(cache, key, fetcher)
except (GitHubSourceError, GiteaSourceError) as exc:
return username, exc
async def collect_merged_activity(
settings: Settings,
cache: FileCache | None,
@@ -120,36 +173,43 @@ async def collect_merged_activity(
) -> ActivityResult:
from_date, to_date, days_count = compute_date_range(options)
gh_key = f"github_{settings.github_username}_{from_date}_{to_date}"
gt_key = f"gitea_{settings.gitea_username}_{from_date}_{to_date}"
github_data: dict[str, int] = {}
gitea_data: dict[str, int] = {}
github_stale = False
gitea_stale = False
if options.source in ("all", "github"):
github_data, github_stale = await _fetch_with_cache(
github_data, github_stale = await _fetch_source_usernames(
cache,
gh_key,
"github",
settings.github_usernames,
from_date,
to_date,
lambda username: (
lambda: fetch_github_activity(
username=settings.github_username,
username=username,
token=settings.github_token,
from_date=from_date,
to_date=to_date,
)
),
)
if options.source in ("all", "gitea"):
gitea_data, gitea_stale = await _fetch_with_cache(
gitea_data, gitea_stale = await _fetch_source_usernames(
cache,
gt_key,
"gitea",
settings.gitea_usernames,
from_date,
to_date,
lambda username: (
lambda: fetch_gitea_activity(
base_url=settings.gitea_base_url,
username=settings.gitea_username,
username=username,
token=settings.gitea_token,
from_date=from_date,
to_date=to_date,
)
),
)
@@ -169,6 +229,10 @@ def _daily_totals(merged: dict[str, dict[str, int]]) -> dict[str, int]:
return {day: int(payload.get("total", 0)) for day, payload in merged.items()}
def _activity_headers(result: ActivityResult) -> dict[str, str]:
return {"X-Activity-Stale": "true" if result.stale else "false"}
def _image_cache_key(prefix: str, options: QueryOptions, result: ActivityResult) -> str:
digest = hashlib.sha1(
json.dumps(result.merged, sort_keys=True, separators=(",", ":")).encode("utf-8")
@@ -203,7 +267,7 @@ async def activity_json(
"stale": result.stale,
"activity": result.merged,
}
return JSONResponse(content=payload)
return JSONResponse(content=payload, headers=_activity_headers(result))
@app.get("/activity.svg")
@@ -222,7 +286,11 @@ async def activity_svg(
if cache is not None:
cached = cache.get_json(cache_key)
if cached is not None:
return Response(content=str(cached.value), media_type="image/svg+xml")
return Response(
content=str(cached.value),
media_type="image/svg+xml",
headers=_activity_headers(result),
)
daily_totals = _daily_totals(result.merged)
total = sum(daily_totals.values())
@@ -236,7 +304,7 @@ async def activity_svg(
)
if cache is not None:
cache.set_json(cache_key, svg)
return Response(content=svg, media_type="image/svg+xml")
return Response(content=svg, media_type="image/svg+xml", headers=_activity_headers(result))
@app.get("/activity.png")
@@ -256,7 +324,11 @@ async def activity_png(
cached = cache.get_json(cache_key)
if cached is not None:
png_data = bytes.fromhex(str(cached.value))
return Response(content=png_data, media_type="image/png")
return Response(
content=png_data,
media_type="image/png",
headers=_activity_headers(result),
)
daily_totals = _daily_totals(result.merged)
total = sum(daily_totals.values())
@@ -275,4 +347,4 @@ async def activity_png(
raise HTTPException(status_code=500, detail="PNG rendering failed") from exc
if cache is not None:
cache.set_json(cache_key, png_data.hex())
return Response(content=png_data, media_type="image/png")
return Response(content=png_data, media_type="image/png", headers=_activity_headers(result))
+28 -1
View File
@@ -1,9 +1,21 @@
from functools import lru_cache
from pydantic import Field
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
def split_usernames(raw: str) -> list[str]:
usernames: list[str] = []
seen: set[str] = set()
for part in raw.split(","):
username = part.strip()
if not username or username in seen:
continue
seen.add(username)
usernames.append(username)
return usernames
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", case_sensitive=False)
@@ -20,6 +32,21 @@ class Settings(BaseSettings):
default_theme: str = Field(default="light", alias="DEFAULT_THEME")
service_title: str = Field(default="git-activity-merge", alias="SERVICE_TITLE")
@field_validator("github_username", "gitea_username")
@classmethod
def _validate_username_list(cls, value: str) -> str:
if not split_usernames(value):
raise ValueError("must contain at least one username")
return value
@property
def github_usernames(self) -> list[str]:
return split_usernames(self.github_username)
@property
def gitea_usernames(self) -> list[str]:
return split_usernames(self.gitea_username)
@lru_cache(maxsize=1)
def get_settings() -> Settings:
+61 -11
View File
@@ -11,6 +11,15 @@ class GitHubSourceError(RuntimeError):
pass
def _iter_year_ranges(from_date: date, to_date: date) -> list[tuple[date, date]]:
ranges: list[tuple[date, date]] = []
for year in range(from_date.year, to_date.year + 1):
range_start = max(from_date, date(year, 1, 1))
range_end = min(to_date, date(year, 12, 31))
ranges.append((range_start, range_end))
return ranges
def _extract_attr(tag: str, attr: str) -> str | None:
match = re.search(rf'{attr}="([^"]+)"', tag)
return match.group(1) if match else None
@@ -42,7 +51,7 @@ def _parse_public_contributions_html(html: str, from_date: date, to_date: date)
return normalized
async def _fetch_github_activity_public(
async def _fetch_github_activity_public_range(
username: str,
from_date: date,
to_date: date,
@@ -66,21 +75,32 @@ async def _fetch_github_activity_public(
return _parse_public_contributions_html(response.text, from_date, to_date)
async def fetch_github_activity(
async def _fetch_github_activity_public(
username: str,
token: str | None,
from_date: date,
to_date: date,
timeout_seconds: float = 20.0,
timeout_seconds: float,
) -> dict[str, int]:
if not token:
return await _fetch_github_activity_public(
normalized: dict[str, int] = {}
for range_start, range_end in _iter_year_ranges(from_date, to_date):
normalized.update(
await _fetch_github_activity_public_range(
username=username,
from_date=from_date,
to_date=to_date,
from_date=range_start,
to_date=range_end,
timeout_seconds=timeout_seconds,
)
)
return normalized
async def _fetch_github_activity_graphql_range(
username: str,
token: str,
from_date: date,
to_date: date,
timeout_seconds: float,
) -> dict[str, int]:
query = """
query($login: String!, $from: DateTime!, $to: DateTime!) {
user(login: $login) {
@@ -103,9 +123,10 @@ async def fetch_github_activity(
"to": datetime.combine(to_date, datetime.max.time(), tzinfo=timezone.utc).isoformat(),
}
headers: dict[str, str] = {"Accept": "application/json"}
if token:
headers["Authorization"] = f"bearer {token}"
headers: dict[str, str] = {
"Accept": "application/json",
"Authorization": f"bearer {token}",
}
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
response = await client.post(
@@ -140,3 +161,32 @@ async def fetch_github_activity(
normalized[date_key] = int(day.get("contributionCount", 0))
return normalized
async def fetch_github_activity(
username: str,
token: str | None,
from_date: date,
to_date: date,
timeout_seconds: float = 20.0,
) -> dict[str, int]:
if not token:
return await _fetch_github_activity_public(
username=username,
from_date=from_date,
to_date=to_date,
timeout_seconds=timeout_seconds,
)
normalized: dict[str, int] = {}
for range_start, range_end in _iter_year_ranges(from_date, to_date):
normalized.update(
await _fetch_github_activity_graphql_range(
username=username,
token=token,
from_date=range_start,
to_date=range_end,
timeout_seconds=timeout_seconds,
)
)
return normalized
+88
View File
@@ -0,0 +1,88 @@
from datetime import date
import pytest
from app.sources import github as github_source
def test_iter_year_ranges_splits_cross_year_window() -> None:
ranges = github_source._iter_year_ranges(date(2025, 6, 1), date(2026, 6, 1))
assert ranges == [
(date(2025, 6, 1), date(2025, 12, 31)),
(date(2026, 1, 1), date(2026, 6, 1)),
]
@pytest.mark.asyncio
async def test_fetch_github_activity_without_token_merges_year_slices(monkeypatch) -> None:
requested_ranges: list[tuple[date, date]] = []
async def fake_fetch_public_range(username, from_date, to_date, timeout_seconds):
requested_ranges.append((from_date, to_date))
return {
from_date.isoformat(): 1,
to_date.isoformat(): 2,
}
monkeypatch.setattr(
github_source,
"_fetch_github_activity_public_range",
fake_fetch_public_range,
)
activity = await github_source.fetch_github_activity(
username="octocat",
token=None,
from_date=date(2025, 6, 1),
to_date=date(2026, 6, 1),
timeout_seconds=1.0,
)
assert requested_ranges == [
(date(2025, 6, 1), date(2025, 12, 31)),
(date(2026, 1, 1), date(2026, 6, 1)),
]
assert activity == {
"2025-06-01": 1,
"2025-12-31": 2,
"2026-01-01": 1,
"2026-06-01": 2,
}
@pytest.mark.asyncio
async def test_fetch_github_activity_with_token_merges_year_slices(monkeypatch) -> None:
requested_ranges: list[tuple[date, date, str]] = []
async def fake_fetch_graphql_range(username, token, from_date, to_date, timeout_seconds):
requested_ranges.append((from_date, to_date, token))
return {
from_date.isoformat(): 3,
to_date.isoformat(): 4,
}
monkeypatch.setattr(
github_source,
"_fetch_github_activity_graphql_range",
fake_fetch_graphql_range,
)
activity = await github_source.fetch_github_activity(
username="octocat",
token="secret",
from_date=date(2025, 6, 1),
to_date=date(2026, 6, 1),
timeout_seconds=1.0,
)
assert requested_ranges == [
(date(2025, 6, 1), date(2025, 12, 31), "secret"),
(date(2026, 1, 1), date(2026, 6, 1), "secret"),
]
assert activity == {
"2025-06-01": 3,
"2025-12-31": 4,
"2026-01-01": 3,
"2026-06-01": 4,
}
+125
View File
@@ -0,0 +1,125 @@
import tempfile
from datetime import date
from pathlib import Path
import pytest
from app.main import QueryOptions, collect_merged_activity, _fetch_source_usernames
from app.cache import FileCache
from app.settings import Settings
from app.sources import github as github_source
@pytest.mark.asyncio
async def test_collect_merged_activity_combines_multiple_usernames(monkeypatch) -> None:
settings = Settings(
_env_file=None,
GITHUB_USERNAME="octocat, hubot",
GITEA_BASE_URL="https://gitea.example.com",
GITEA_USERNAME="alice, bob",
)
github_calls: list[str] = []
gitea_calls: list[str] = []
async def fake_fetch_github_activity(username, token, from_date, to_date, timeout_seconds=20.0):
github_calls.append(username)
return {"2026-01-01": 1 if username == "octocat" else 2}
async def fake_fetch_gitea_activity(base_url, username, token, from_date, to_date, timeout_seconds=20.0):
gitea_calls.append(username)
return {"2026-01-01": 3 if username == "alice" else 4}
monkeypatch.setattr("app.main.fetch_github_activity", fake_fetch_github_activity)
monkeypatch.setattr("app.main.fetch_gitea_activity", fake_fetch_gitea_activity)
result = await collect_merged_activity(
settings=settings,
cache=None,
options=QueryOptions(year=None, days=1, theme="light", source="all"),
)
assert sorted(github_calls) == ["hubot", "octocat"]
assert sorted(gitea_calls) == ["alice", "bob"]
assert result.merged["2026-01-01"] == {"github": 3, "gitea": 7, "total": 10}
@pytest.mark.asyncio
async def test_fetch_source_usernames_skips_failed_accounts() -> None:
async def ok_fetch():
return {"2026-01-01": 2}
async def bad_fetch():
raise github_source.GitHubSourceError("boom")
result, stale = await _fetch_source_usernames(
None,
"github",
["octocat", "hubot"],
date(2026, 1, 1),
date(2026, 1, 1),
lambda username: ok_fetch if username == "octocat" else bad_fetch,
)
assert result == {"2026-01-01": 2}
assert stale is False
@pytest.mark.asyncio
async def test_fetch_source_usernames_does_not_swallow_programming_errors() -> None:
def broken_fetcher(username):
async def fetch():
raise TypeError("bad shape")
return fetch
with pytest.raises(TypeError, match="bad shape"):
await _fetch_source_usernames(
None,
"github",
["octocat", "hubot"],
date(2026, 1, 1),
date(2026, 1, 1),
broken_fetcher,
)
@pytest.mark.asyncio
async def test_fetch_source_usernames_keeps_cache_after_username_list_changes() -> None:
with tempfile.TemporaryDirectory(dir=Path.cwd()) as temp_dir:
cache = FileCache(cache_dir=temp_dir, default_ttl_seconds=0)
def prime_fetcher(username):
async def fetch():
return {"2026-01-01": 1}
return fetch
await _fetch_source_usernames(
cache,
"github",
["octocat"],
date(2026, 1, 1),
date(2026, 1, 1),
prime_fetcher,
)
def fetch_after_change(username):
async def fetch():
if username == "octocat":
raise github_source.GitHubSourceError("down")
return {"2026-01-01": 2}
return fetch
result, stale = await _fetch_source_usernames(
cache,
"github",
["octocat", "hubot"],
date(2026, 1, 1),
date(2026, 1, 1),
fetch_after_change,
)
assert result == {"2026-01-01": 3}
assert stale is True
+39
View File
@@ -8,6 +8,31 @@ from app.main import ActivityResult, app
from app.settings import get_settings
def test_activity_json_reports_stale_header(monkeypatch) -> None:
monkeypatch.setenv("GITHUB_USERNAME", "octocat")
monkeypatch.setenv("GITEA_BASE_URL", "https://gitea.example.com")
monkeypatch.setenv("GITEA_USERNAME", "octocat")
get_settings.cache_clear()
async def fake_collect_merged_activity(settings, cache, options):
return ActivityResult(
merged={"2026-01-01": {"github": 1, "gitea": 2, "total": 3}},
stale=True,
from_date=date(2026, 1, 1),
to_date=date(2026, 1, 1),
days_count=1,
)
monkeypatch.setattr("app.main.collect_merged_activity", fake_collect_merged_activity)
client = TestClient(app)
response = client.get("/activity.json")
assert response.status_code == 200
assert response.headers["x-activity-stale"] == "true"
assert response.json()["stale"] is True
def test_activity_svg_returns_svg_content_type(monkeypatch) -> None:
monkeypatch.setenv("GITHUB_USERNAME", "octocat")
monkeypatch.setenv("GITEA_BASE_URL", "https://gitea.example.com")
@@ -35,4 +60,18 @@ def test_activity_svg_returns_svg_content_type(monkeypatch) -> None:
assert response.status_code == 200
assert response.headers["content-type"].startswith("image/svg+xml")
assert response.headers["x-activity-stale"] == "false"
assert "<svg" in response.text
def test_activity_json_rejects_year_and_days_together(monkeypatch) -> None:
monkeypatch.setenv("GITHUB_USERNAME", "octocat")
monkeypatch.setenv("GITEA_BASE_URL", "https://gitea.example.com")
monkeypatch.setenv("GITEA_USERNAME", "octocat")
get_settings.cache_clear()
client = TestClient(app)
response = client.get("/activity.json?year=2026&days=30")
assert response.status_code == 400
assert response.json() == {"detail": "Provide either year or days, not both"}
+9
View File
@@ -0,0 +1,9 @@
from app.settings import split_usernames
def test_split_usernames_trims_and_deduplicates() -> None:
assert split_usernames("octocat, hubot, octocat, , gitea-user") == [
"octocat",
"hubot",
"gitea-user",
]