Harden multi-user activity fetching
ci / deploy-coolify (push) Successful in 7s
ci / test (push) Successful in 13s
ci / push-image (push) Successful in 55s

This commit is contained in:
Space-Banane
2026-06-04 22:51:48 +02:00
parent 4444174365
commit c622420e38
2 changed files with 97 additions and 6 deletions
+30 -5
View File
@@ -125,22 +125,47 @@ async def _fetch_source_usernames(
if not usernames:
return {}, False
cache_usernames = hashlib.sha1(",".join(sorted(usernames)).encode("utf-8")).hexdigest()[:12]
fetch_tasks: list[asyncio.Task[tuple[dict[str, int], bool]]] = []
fetch_tasks: list[asyncio.Task[tuple[str, tuple[dict[str, int], bool] | Exception]]] = []
for username in usernames:
key = f"{source_prefix}_{cache_usernames}_{username}_{from_date}_{to_date}"
fetch_tasks.append(asyncio.create_task(_fetch_with_cache(cache, key, fetcher_factory(username))))
key = f"{source_prefix}_{username}_{from_date}_{to_date}"
fetch_tasks.append(
asyncio.create_task(
_fetch_source_username(cache, key, username, fetcher_factory(username))
)
)
combined: dict[str, int] = {}
stale = False
for data, entry_stale in await asyncio.gather(*fetch_tasks):
failures: list[Exception] = []
for username, result in await asyncio.gather(*fetch_tasks):
if isinstance(result, Exception):
logger.warning("source fetch failed for %s/%s, skipping user: %s", source_prefix, username, result)
failures.append(result)
continue
data, entry_stale = result
stale = stale or entry_stale
for day, count in data.items():
combined[day] = combined.get(day, 0) + int(count)
if not combined and failures:
raise failures[0]
return combined, stale
async def _fetch_source_username(
cache: FileCache | None,
key: str,
username: str,
fetcher,
) -> tuple[str, tuple[dict[str, int], bool] | Exception]:
try:
return username, await _fetch_with_cache(cache, key, fetcher)
except Exception as exc:
return username, exc
async def collect_merged_activity(
settings: Settings,
cache: FileCache | None,