feat: add rate limiting, retry with backoff, and max count cap

- Add configurable request delay between paginated API calls (default 1.5s)
- Add retry with exponential backoff on HTTP 429 and Twitter error code 88
- Add hard max count cap (default 200, absolute ceiling 500)
- Add rateLimit config section with requestDelay, maxRetries, retryBaseDelay, maxCount
- Add normalization tests for rateLimit config
This commit is contained in:
jackwener
2026-03-07 19:02:49 +08:00
parent 0f26e20abb
commit 55c48b077b
6 changed files with 125 additions and 31 deletions

View File

@@ -60,15 +60,16 @@ def _load_tweets_from_json(path):
raise RuntimeError("Invalid tweet JSON file %s: %s" % (path, exc))
def _get_client():
# type: () -> TwitterClient
def _get_client(config=None):
# type: (Optional[Dict[str, Any]]) -> TwitterClient
"""Create an authenticated API client."""
console.print("\n🔐 Getting Twitter cookies...")
try:
cookies = get_cookies()
except RuntimeError as exc:
raise RuntimeError(str(exc))
return TwitterClient(cookies["auth_token"], cookies["ct0"])
rate_limit_config = (config or {}).get("rateLimit")
return TwitterClient(cookies["auth_token"], cookies["ct0"], rate_limit_config)
def _resolve_fetch_count(max_count, configured):
@@ -128,7 +129,7 @@ def feed(feed_type, max_count, as_json, input_file, output_file, do_filter):
console.print(" Loaded %d tweets" % len(tweets))
else:
fetch_count = _resolve_fetch_count(max_count, config.get("fetch", {}).get("count", 50))
client = _get_client()
client = _get_client(config)
label = "following feed" if feed_type == "following" else "home timeline"
console.print("📡 Fetching %s (%d tweets)...\n" % (label, fetch_count))
start = time.time()
@@ -169,7 +170,7 @@ def favorite(max_count, as_json, output_file, do_filter):
config = load_config()
try:
fetch_count = _resolve_fetch_count(max_count, config.get("fetch", {}).get("count", 50))
client = _get_client()
client = _get_client(config)
console.print("🔖 Fetching favorites (%d tweets)...\n" % fetch_count)
start = time.time()
tweets = client.fetch_bookmarks(fetch_count)
@@ -199,8 +200,9 @@ def user(screen_name):
# type: (str,) -> None
"""View a user's profile. SCREEN_NAME is the @handle (without @)."""
screen_name = screen_name.lstrip("@")
config = load_config()
try:
client = _get_client()
client = _get_client(config)
console.print("👤 Fetching user @%s..." % screen_name)
profile = client.fetch_user(screen_name)
except RuntimeError as exc:
@@ -219,9 +221,10 @@ def user_posts(screen_name, max_count, as_json):
# type: (str, int, bool) -> None
"""List a user's tweets. SCREEN_NAME is the @handle (without @)."""
screen_name = screen_name.lstrip("@")
config = load_config()
try:
fetch_count = _resolve_fetch_count(max_count, 20)
client = _get_client()
client = _get_client(config)
console.print("👤 Fetching @%s's profile..." % screen_name)
profile = client.fetch_user(screen_name)
console.print("📝 Fetching tweets (%d)...\n" % fetch_count)