luna/twitter-cli-cookiefile
1
0
Fork 0
Code Issues Pull Requests Actions Releases Activity

Feed cursor pagination (#49)

Browse Source 
* Expose promoted tweets in feed output

* Add cursor-based feed pagination output
This commit is contained in:
Lucius
2026-04-10 01:20:18 +08:00
committed by GitHub
parent e3545ab069
commit 7816f8d813
12 changed files with 199 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
twitter_cli/cli.py
View File

@@ -349,6 +349,15 @@ def _fetch_and_display(fetch_fn, label, emoji, max_count, as_json, as_yaml, outp
console.print()
def _emit_timeline_structured(tweets, next_cursor, *, as_json, as_yaml):
# type: (TweetList, Optional[str], bool, bool) -> bool
"""Emit timeline data with pagination metadata while keeping `data` a tweet list."""
payload = success_payload(tweets_to_data(tweets))
if next_cursor:
payload["pagination"] = {"nextCursor": next_cursor}
return emit_structured(payload, as_json=as_json, as_yaml=as_yaml)
def _run_bookmarks_command(max_count, as_json, as_yaml, output_file, do_filter, compact=False, full_text=False):
# type: (Optional[int], bool, bool, Optional[str], bool, bool, bool) -> None
config = load_config()
@@ -401,17 +410,24 @@ def _inherit_flag(ctx, name, value):
help="Feed type: for-you (algorithmic) or following (chronological).",
)
@click.option("--max", "-n", "max_count", type=int, default=None, help="Max number of tweets to fetch.")
@click.option("--cursor", type=str, default=None, help="Pagination cursor for continuing a previous feed request.")
@structured_output_options
@click.option("--input", "-i", "input_file", type=str, default=None, help="Load tweets from JSON file.")
@click.option("--output", "-o", "output_file", type=str, default=None, help="Save filtered tweets to JSON file.")
@click.option("--filter", "do_filter", is_flag=True, help="Enable score-based filtering.")
@click.option("--full-text", is_flag=True, help="Show full tweet text in table output.")
@click.option(
"--include-promoted/--no-include-promoted",
default=False,
help="Include promoted tweets when the timeline endpoint exposes them.",
)
@click.pass_context
def feed(ctx, feed_type, max_count, as_json, as_yaml, input_file, output_file, do_filter, full_text):
# type: (Any, str, Optional[int], bool, bool, Optional[str], Optional[str], bool, bool) -> None
def feed(ctx, feed_type, max_count, cursor, as_json, as_yaml, input_file, output_file, do_filter, full_text, include_promoted):
# type: (Any, str, Optional[int], Optional[str], bool, bool, Optional[str], Optional[str], bool, bool, bool) -> None
"""Fetch home timeline with optional filtering."""
compact = ctx.obj.get("compact", False)
rich_output = use_rich_output(as_json=as_json, as_yaml=as_yaml, compact=compact)
next_cursor = None # type: Optional[str]
config = load_config()
try:
if input_file:
@@ -428,9 +444,19 @@ def feed(ctx, feed_type, max_count, as_json, as_yaml, input_file, output_file, d
console.print("📡 Fetching %s (%d tweets)...\n" % (label, fetch_count))
start = time.time()
if feed_type == "following":
tweets = client.fetch_following_feed(fetch_count)
tweets, next_cursor = client.fetch_following_feed(
fetch_count,
include_promoted=include_promoted,
cursor=cursor,
return_cursor=True,
)
else:
tweets = client.fetch_home_timeline(fetch_count)
tweets, next_cursor = client.fetch_home_timeline(
fetch_count,
include_promoted=include_promoted,
cursor=cursor,
return_cursor=True,
)
elapsed = time.time() - start
if rich_output:
console.print("✅ Fetched %d tweets in %.1fs\n" % (len(tweets), elapsed))
@@ -450,7 +476,7 @@ def feed(ctx, feed_type, max_count, as_json, as_yaml, input_file, output_file, d
save_tweet_cache(filtered)
if emit_structured(tweets_to_data(filtered), as_json=as_json, as_yaml=as_yaml):
if _emit_timeline_structured(filtered, next_cursor, as_json=as_json, as_yaml=as_yaml):
return
title = "👥 Following" if feed_type == "following" else "📱 Twitter"

28
twitter_cli/client.py
View File

@@ -153,22 +153,28 @@ class TwitterClient:
# ── Read operations ──────────────────────────────────────────────
def fetch_home_timeline(self, count=20):
# type: (int) -> List[Tweet]
def fetch_home_timeline(self, count=20, include_promoted=False, cursor=None, return_cursor=False):
# type: (int, bool, Optional[str], bool) -> Any
"""Fetch home timeline tweets."""
return self._fetch_timeline(
"HomeTimeline",
count,
lambda data: _deep_get(data, "data", "home", "home_timeline_urt", "instructions"),
include_promoted=include_promoted,
start_cursor=cursor,
return_cursor=return_cursor,
)
def fetch_following_feed(self, count=20):
# type: (int) -> List[Tweet]
def fetch_following_feed(self, count=20, include_promoted=False, cursor=None, return_cursor=False):
# type: (int, bool, Optional[str], bool) -> Any
"""Fetch chronological following feed."""
return self._fetch_timeline(
"HomeLatestTimeline",
count,
lambda data: _deep_get(data, "data", "home", "home_timeline_urt", "instructions"),
include_promoted=include_promoted,
start_cursor=cursor,
return_cursor=return_cursor,
)
def fetch_bookmarks(self, count=50):
@@ -732,8 +738,8 @@ class TwitterClient:
# ── Internal: timeline / user list fetchers ──────────────────────
def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None, use_post=False):
# type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]], bool) -> List[Tweet]
def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None, use_post=False, include_promoted=False, start_cursor=None, return_cursor=False):
# type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]], bool, bool, Optional[str], bool) -> Any
"""Generic timeline fetcher with pagination and deduplication.
Args:
@@ -751,7 +757,8 @@ class TwitterClient:
tweets = [] # type: List[Tweet]
seen_ids = set() # type: Set[str]
cursor = None # type: Optional[str]
cursor = start_cursor # type: Optional[str]
continuation_cursor = None # type: Optional[str]
attempts = 0
max_attempts = int(math.ceil(count / 20.0)) + 2
@@ -763,7 +770,7 @@ class TwitterClient:
else:
variables = {
"count": min(count - len(tweets) + 5, 40),
"includePromotedContent": False,
"includePromotedContent": include_promoted,
"latestControlAvailable": True,
"requestContext": "launch",
}
@@ -784,10 +791,13 @@ class TwitterClient:
tweets.append(tweet)
if not next_cursor:
continuation_cursor = None
break
if next_cursor == cursor:
logger.debug("Timeline pagination stopped because cursor did not advance: %s", next_cursor)
continuation_cursor = None
break
continuation_cursor = next_cursor
cursor = next_cursor
if not new_tweets:
@@ -799,6 +809,8 @@ class TwitterClient:
logger.debug("Sleeping %.1fs between requests", jitter)
time.sleep(jitter)
if return_cursor:
return tweets[:count], continuation_cursor
return tweets[:count]
def _fetch_user_list(self, operation_name, user_id, count, get_instructions):

1
twitter_cli/models.py
View File

@@ -53,6 +53,7 @@ class Tweet:
article_title: Optional[str] = None
article_text: Optional[str] = None
is_subscriber_only: bool = False
is_promoted: bool = False
@dataclass

9
twitter_cli/parser.py
View File

@@ -504,6 +504,10 @@ def parse_timeline_response(data, get_instructions):
if result:
tweet = parse_tweet_result(result)
if tweet:
tweet.is_promoted = bool(
str(entry.get("entryId") or "").startswith("promoted-")
or item_content.get("promotedMetadata")
)
tweets.append(tweet)
for nested_item in content.get("items", []):
@@ -517,6 +521,11 @@ def parse_timeline_response(data, get_instructions):
if nested_result:
tweet = parse_tweet_result(nested_result)
if tweet:
nested_item_content = _deep_get(nested_item, "item", "itemContent") or {}
tweet.is_promoted = bool(
str(_deep_get(nested_item, "entryId") or "").startswith("promoted-")
or nested_item_content.get("promotedMetadata")
)
tweets.append(tweet)
return tweets, next_cursor

2
twitter_cli/serialization.py
View File

@@ -47,6 +47,7 @@ def tweet_to_dict(tweet: Tweet) -> Dict[str, Any]:
"lang": tweet.lang,
"score": tweet.score,
"isSubscriberOnly": tweet.is_subscriber_only,
"isPromoted": tweet.is_promoted,
}
if tweet.article_title is not None:
data["articleTitle"] = tweet.article_title
@@ -124,6 +125,7 @@ def tweet_from_dict(data: Dict[str, Any]) -> Tweet:
article_title=_optional_str(data.get("articleTitle")),
article_text=_optional_str(data.get("articleText")),
is_subscriber_only=bool(data.get("isSubscriberOnly", False)),
is_promoted=bool(data.get("isPromoted", False)),
)