Harden multi-user activity fetching
ci / deploy-coolify (push) Successful in 7s
ci / test (push) Successful in 13s
ci / push-image (push) Successful in 55s

This commit is contained in:
Space-Banane
2026-06-04 22:51:48 +02:00
parent 4444174365
commit c622420e38
2 changed files with 97 additions and 6 deletions
+30 -5
View File
@@ -125,22 +125,47 @@ async def _fetch_source_usernames(
if not usernames:
return {}, False
cache_usernames = hashlib.sha1(",".join(sorted(usernames)).encode("utf-8")).hexdigest()[:12]
fetch_tasks: list[asyncio.Task[tuple[dict[str, int], bool]]] = []
fetch_tasks: list[asyncio.Task[tuple[str, tuple[dict[str, int], bool] | Exception]]] = []
for username in usernames:
key = f"{source_prefix}_{cache_usernames}_{username}_{from_date}_{to_date}"
fetch_tasks.append(asyncio.create_task(_fetch_with_cache(cache, key, fetcher_factory(username))))
key = f"{source_prefix}_{username}_{from_date}_{to_date}"
fetch_tasks.append(
asyncio.create_task(
_fetch_source_username(cache, key, username, fetcher_factory(username))
)
)
combined: dict[str, int] = {}
stale = False
for data, entry_stale in await asyncio.gather(*fetch_tasks):
failures: list[Exception] = []
for username, result in await asyncio.gather(*fetch_tasks):
if isinstance(result, Exception):
logger.warning("source fetch failed for %s/%s, skipping user: %s", source_prefix, username, result)
failures.append(result)
continue
data, entry_stale = result
stale = stale or entry_stale
for day, count in data.items():
combined[day] = combined.get(day, 0) + int(count)
if not combined and failures:
raise failures[0]
return combined, stale
async def _fetch_source_username(
cache: FileCache | None,
key: str,
username: str,
fetcher,
) -> tuple[str, tuple[dict[str, int], bool] | Exception]:
try:
return username, await _fetch_with_cache(cache, key, fetcher)
except Exception as exc:
return username, exc
async def collect_merged_activity(
settings: Settings,
cache: FileCache | None,
+67 -1
View File
@@ -1,9 +1,13 @@
import tempfile
from datetime import date
from pathlib import Path
import pytest
from app.main import QueryOptions, collect_merged_activity
from app.main import QueryOptions, collect_merged_activity, _fetch_source_usernames
from app.cache import FileCache
from app.settings import Settings
from app.sources import github as github_source
@pytest.mark.asyncio
@@ -38,3 +42,65 @@ async def test_collect_merged_activity_combines_multiple_usernames(monkeypatch)
assert sorted(github_calls) == ["hubot", "octocat"]
assert sorted(gitea_calls) == ["alice", "bob"]
assert result.merged["2026-01-01"] == {"github": 3, "gitea": 7, "total": 10}
@pytest.mark.asyncio
async def test_fetch_source_usernames_skips_failed_accounts() -> None:
async def ok_fetch():
return {"2026-01-01": 2}
async def bad_fetch():
raise github_source.GitHubSourceError("boom")
result, stale = await _fetch_source_usernames(
None,
"github",
["octocat", "hubot"],
date(2026, 1, 1),
date(2026, 1, 1),
lambda username: ok_fetch if username == "octocat" else bad_fetch,
)
assert result == {"2026-01-01": 2}
assert stale is False
@pytest.mark.asyncio
async def test_fetch_source_usernames_keeps_cache_after_username_list_changes() -> None:
with tempfile.TemporaryDirectory(dir=Path.cwd()) as temp_dir:
cache = FileCache(cache_dir=temp_dir, default_ttl_seconds=0)
def prime_fetcher(username):
async def fetch():
return {"2026-01-01": 1}
return fetch
await _fetch_source_usernames(
cache,
"github",
["octocat"],
date(2026, 1, 1),
date(2026, 1, 1),
prime_fetcher,
)
def fetch_after_change(username):
async def fetch():
if username == "octocat":
raise github_source.GitHubSourceError("down")
return {"2026-01-01": 2}
return fetch
result, stale = await _fetch_source_usernames(
cache,
"github",
["octocat", "hubot"],
date(2026, 1, 1),
date(2026, 1, 1),
fetch_after_change,
)
assert result == {"2026-01-01": 3}
assert stale is True