Harden multi-user activity fetching
This commit is contained in:
+30
-5
@@ -125,22 +125,47 @@ async def _fetch_source_usernames(
|
||||
if not usernames:
|
||||
return {}, False
|
||||
|
||||
cache_usernames = hashlib.sha1(",".join(sorted(usernames)).encode("utf-8")).hexdigest()[:12]
|
||||
fetch_tasks: list[asyncio.Task[tuple[dict[str, int], bool]]] = []
|
||||
fetch_tasks: list[asyncio.Task[tuple[str, tuple[dict[str, int], bool] | Exception]]] = []
|
||||
for username in usernames:
|
||||
key = f"{source_prefix}_{cache_usernames}_{username}_{from_date}_{to_date}"
|
||||
fetch_tasks.append(asyncio.create_task(_fetch_with_cache(cache, key, fetcher_factory(username))))
|
||||
key = f"{source_prefix}_{username}_{from_date}_{to_date}"
|
||||
fetch_tasks.append(
|
||||
asyncio.create_task(
|
||||
_fetch_source_username(cache, key, username, fetcher_factory(username))
|
||||
)
|
||||
)
|
||||
|
||||
combined: dict[str, int] = {}
|
||||
stale = False
|
||||
for data, entry_stale in await asyncio.gather(*fetch_tasks):
|
||||
failures: list[Exception] = []
|
||||
for username, result in await asyncio.gather(*fetch_tasks):
|
||||
if isinstance(result, Exception):
|
||||
logger.warning("source fetch failed for %s/%s, skipping user: %s", source_prefix, username, result)
|
||||
failures.append(result)
|
||||
continue
|
||||
|
||||
data, entry_stale = result
|
||||
stale = stale or entry_stale
|
||||
for day, count in data.items():
|
||||
combined[day] = combined.get(day, 0) + int(count)
|
||||
|
||||
if not combined and failures:
|
||||
raise failures[0]
|
||||
|
||||
return combined, stale
|
||||
|
||||
|
||||
async def _fetch_source_username(
|
||||
cache: FileCache | None,
|
||||
key: str,
|
||||
username: str,
|
||||
fetcher,
|
||||
) -> tuple[str, tuple[dict[str, int], bool] | Exception]:
|
||||
try:
|
||||
return username, await _fetch_with_cache(cache, key, fetcher)
|
||||
except Exception as exc:
|
||||
return username, exc
|
||||
|
||||
|
||||
async def collect_merged_activity(
|
||||
settings: Settings,
|
||||
cache: FileCache | None,
|
||||
|
||||
+67
-1
@@ -1,9 +1,13 @@
|
||||
import tempfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.main import QueryOptions, collect_merged_activity
|
||||
from app.main import QueryOptions, collect_merged_activity, _fetch_source_usernames
|
||||
from app.cache import FileCache
|
||||
from app.settings import Settings
|
||||
from app.sources import github as github_source
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -38,3 +42,65 @@ async def test_collect_merged_activity_combines_multiple_usernames(monkeypatch)
|
||||
assert sorted(github_calls) == ["hubot", "octocat"]
|
||||
assert sorted(gitea_calls) == ["alice", "bob"]
|
||||
assert result.merged["2026-01-01"] == {"github": 3, "gitea": 7, "total": 10}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_source_usernames_skips_failed_accounts() -> None:
|
||||
async def ok_fetch():
|
||||
return {"2026-01-01": 2}
|
||||
|
||||
async def bad_fetch():
|
||||
raise github_source.GitHubSourceError("boom")
|
||||
|
||||
result, stale = await _fetch_source_usernames(
|
||||
None,
|
||||
"github",
|
||||
["octocat", "hubot"],
|
||||
date(2026, 1, 1),
|
||||
date(2026, 1, 1),
|
||||
lambda username: ok_fetch if username == "octocat" else bad_fetch,
|
||||
)
|
||||
|
||||
assert result == {"2026-01-01": 2}
|
||||
assert stale is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_source_usernames_keeps_cache_after_username_list_changes() -> None:
|
||||
with tempfile.TemporaryDirectory(dir=Path.cwd()) as temp_dir:
|
||||
cache = FileCache(cache_dir=temp_dir, default_ttl_seconds=0)
|
||||
|
||||
def prime_fetcher(username):
|
||||
async def fetch():
|
||||
return {"2026-01-01": 1}
|
||||
|
||||
return fetch
|
||||
|
||||
await _fetch_source_usernames(
|
||||
cache,
|
||||
"github",
|
||||
["octocat"],
|
||||
date(2026, 1, 1),
|
||||
date(2026, 1, 1),
|
||||
prime_fetcher,
|
||||
)
|
||||
|
||||
def fetch_after_change(username):
|
||||
async def fetch():
|
||||
if username == "octocat":
|
||||
raise github_source.GitHubSourceError("down")
|
||||
return {"2026-01-01": 2}
|
||||
|
||||
return fetch
|
||||
|
||||
result, stale = await _fetch_source_usernames(
|
||||
cache,
|
||||
"github",
|
||||
["octocat", "hubot"],
|
||||
date(2026, 1, 1),
|
||||
date(2026, 1, 1),
|
||||
fetch_after_change,
|
||||
)
|
||||
|
||||
assert result == {"2026-01-01": 3}
|
||||
assert stale is True
|
||||
|
||||
Reference in New Issue
Block a user