Harden multi-user activity fetching
This commit is contained in:
+30
-5
@@ -125,22 +125,47 @@ async def _fetch_source_usernames(
|
|||||||
if not usernames:
|
if not usernames:
|
||||||
return {}, False
|
return {}, False
|
||||||
|
|
||||||
cache_usernames = hashlib.sha1(",".join(sorted(usernames)).encode("utf-8")).hexdigest()[:12]
|
fetch_tasks: list[asyncio.Task[tuple[str, tuple[dict[str, int], bool] | Exception]]] = []
|
||||||
fetch_tasks: list[asyncio.Task[tuple[dict[str, int], bool]]] = []
|
|
||||||
for username in usernames:
|
for username in usernames:
|
||||||
key = f"{source_prefix}_{cache_usernames}_{username}_{from_date}_{to_date}"
|
key = f"{source_prefix}_{username}_{from_date}_{to_date}"
|
||||||
fetch_tasks.append(asyncio.create_task(_fetch_with_cache(cache, key, fetcher_factory(username))))
|
fetch_tasks.append(
|
||||||
|
asyncio.create_task(
|
||||||
|
_fetch_source_username(cache, key, username, fetcher_factory(username))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
combined: dict[str, int] = {}
|
combined: dict[str, int] = {}
|
||||||
stale = False
|
stale = False
|
||||||
for data, entry_stale in await asyncio.gather(*fetch_tasks):
|
failures: list[Exception] = []
|
||||||
|
for username, result in await asyncio.gather(*fetch_tasks):
|
||||||
|
if isinstance(result, Exception):
|
||||||
|
logger.warning("source fetch failed for %s/%s, skipping user: %s", source_prefix, username, result)
|
||||||
|
failures.append(result)
|
||||||
|
continue
|
||||||
|
|
||||||
|
data, entry_stale = result
|
||||||
stale = stale or entry_stale
|
stale = stale or entry_stale
|
||||||
for day, count in data.items():
|
for day, count in data.items():
|
||||||
combined[day] = combined.get(day, 0) + int(count)
|
combined[day] = combined.get(day, 0) + int(count)
|
||||||
|
|
||||||
|
if not combined and failures:
|
||||||
|
raise failures[0]
|
||||||
|
|
||||||
return combined, stale
|
return combined, stale
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch_source_username(
|
||||||
|
cache: FileCache | None,
|
||||||
|
key: str,
|
||||||
|
username: str,
|
||||||
|
fetcher,
|
||||||
|
) -> tuple[str, tuple[dict[str, int], bool] | Exception]:
|
||||||
|
try:
|
||||||
|
return username, await _fetch_with_cache(cache, key, fetcher)
|
||||||
|
except Exception as exc:
|
||||||
|
return username, exc
|
||||||
|
|
||||||
|
|
||||||
async def collect_merged_activity(
|
async def collect_merged_activity(
|
||||||
settings: Settings,
|
settings: Settings,
|
||||||
cache: FileCache | None,
|
cache: FileCache | None,
|
||||||
|
|||||||
+67
-1
@@ -1,9 +1,13 @@
|
|||||||
|
import tempfile
|
||||||
from datetime import date
|
from datetime import date
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.main import QueryOptions, collect_merged_activity
|
from app.main import QueryOptions, collect_merged_activity, _fetch_source_usernames
|
||||||
|
from app.cache import FileCache
|
||||||
from app.settings import Settings
|
from app.settings import Settings
|
||||||
|
from app.sources import github as github_source
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -38,3 +42,65 @@ async def test_collect_merged_activity_combines_multiple_usernames(monkeypatch)
|
|||||||
assert sorted(github_calls) == ["hubot", "octocat"]
|
assert sorted(github_calls) == ["hubot", "octocat"]
|
||||||
assert sorted(gitea_calls) == ["alice", "bob"]
|
assert sorted(gitea_calls) == ["alice", "bob"]
|
||||||
assert result.merged["2026-01-01"] == {"github": 3, "gitea": 7, "total": 10}
|
assert result.merged["2026-01-01"] == {"github": 3, "gitea": 7, "total": 10}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_source_usernames_skips_failed_accounts() -> None:
|
||||||
|
async def ok_fetch():
|
||||||
|
return {"2026-01-01": 2}
|
||||||
|
|
||||||
|
async def bad_fetch():
|
||||||
|
raise github_source.GitHubSourceError("boom")
|
||||||
|
|
||||||
|
result, stale = await _fetch_source_usernames(
|
||||||
|
None,
|
||||||
|
"github",
|
||||||
|
["octocat", "hubot"],
|
||||||
|
date(2026, 1, 1),
|
||||||
|
date(2026, 1, 1),
|
||||||
|
lambda username: ok_fetch if username == "octocat" else bad_fetch,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {"2026-01-01": 2}
|
||||||
|
assert stale is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_source_usernames_keeps_cache_after_username_list_changes() -> None:
|
||||||
|
with tempfile.TemporaryDirectory(dir=Path.cwd()) as temp_dir:
|
||||||
|
cache = FileCache(cache_dir=temp_dir, default_ttl_seconds=0)
|
||||||
|
|
||||||
|
def prime_fetcher(username):
|
||||||
|
async def fetch():
|
||||||
|
return {"2026-01-01": 1}
|
||||||
|
|
||||||
|
return fetch
|
||||||
|
|
||||||
|
await _fetch_source_usernames(
|
||||||
|
cache,
|
||||||
|
"github",
|
||||||
|
["octocat"],
|
||||||
|
date(2026, 1, 1),
|
||||||
|
date(2026, 1, 1),
|
||||||
|
prime_fetcher,
|
||||||
|
)
|
||||||
|
|
||||||
|
def fetch_after_change(username):
|
||||||
|
async def fetch():
|
||||||
|
if username == "octocat":
|
||||||
|
raise github_source.GitHubSourceError("down")
|
||||||
|
return {"2026-01-01": 2}
|
||||||
|
|
||||||
|
return fetch
|
||||||
|
|
||||||
|
result, stale = await _fetch_source_usernames(
|
||||||
|
cache,
|
||||||
|
"github",
|
||||||
|
["octocat", "hubot"],
|
||||||
|
date(2026, 1, 1),
|
||||||
|
date(2026, 1, 1),
|
||||||
|
fetch_after_change,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {"2026-01-01": 3}
|
||||||
|
assert stale is True
|
||||||
|
|||||||
Reference in New Issue
Block a user