Harden multi-user activity fetching
This commit is contained in:
+30
-5
@@ -125,22 +125,47 @@ async def _fetch_source_usernames(
|
||||
if not usernames:
|
||||
return {}, False
|
||||
|
||||
cache_usernames = hashlib.sha1(",".join(sorted(usernames)).encode("utf-8")).hexdigest()[:12]
|
||||
fetch_tasks: list[asyncio.Task[tuple[dict[str, int], bool]]] = []
|
||||
fetch_tasks: list[asyncio.Task[tuple[str, tuple[dict[str, int], bool] | Exception]]] = []
|
||||
for username in usernames:
|
||||
key = f"{source_prefix}_{cache_usernames}_{username}_{from_date}_{to_date}"
|
||||
fetch_tasks.append(asyncio.create_task(_fetch_with_cache(cache, key, fetcher_factory(username))))
|
||||
key = f"{source_prefix}_{username}_{from_date}_{to_date}"
|
||||
fetch_tasks.append(
|
||||
asyncio.create_task(
|
||||
_fetch_source_username(cache, key, username, fetcher_factory(username))
|
||||
)
|
||||
)
|
||||
|
||||
combined: dict[str, int] = {}
|
||||
stale = False
|
||||
for data, entry_stale in await asyncio.gather(*fetch_tasks):
|
||||
failures: list[Exception] = []
|
||||
for username, result in await asyncio.gather(*fetch_tasks):
|
||||
if isinstance(result, Exception):
|
||||
logger.warning("source fetch failed for %s/%s, skipping user: %s", source_prefix, username, result)
|
||||
failures.append(result)
|
||||
continue
|
||||
|
||||
data, entry_stale = result
|
||||
stale = stale or entry_stale
|
||||
for day, count in data.items():
|
||||
combined[day] = combined.get(day, 0) + int(count)
|
||||
|
||||
if not combined and failures:
|
||||
raise failures[0]
|
||||
|
||||
return combined, stale
|
||||
|
||||
|
||||
async def _fetch_source_username(
|
||||
cache: FileCache | None,
|
||||
key: str,
|
||||
username: str,
|
||||
fetcher,
|
||||
) -> tuple[str, tuple[dict[str, int], bool] | Exception]:
|
||||
try:
|
||||
return username, await _fetch_with_cache(cache, key, fetcher)
|
||||
except Exception as exc:
|
||||
return username, exc
|
||||
|
||||
|
||||
async def collect_merged_activity(
|
||||
settings: Settings,
|
||||
cache: FileCache | None,
|
||||
|
||||
Reference in New Issue
Block a user