fix: SearchTimeline POST + ondemand URL guard + refresh queryIds

Twitter migrated SearchTimeline from GET to POST; update fetch_search
to route through _graphql_post via a new use_post flag on _fetch_timeline,
avoiding duplicating the pagination logic (closes #39, refs #40, #42).

Also guard against get_ondemand_file_url returning None so the error
message is clear instead of crashing on NoneType.split (closes #43-adjacent).

Refresh all FALLBACK_QUERY_IDS from live JS bundles — 18 operations updated.
This commit is contained in:
jackwener
2026-03-20 18:21:29 +08:00
parent 7b8a7dd5de
commit 199a1490f9
3 changed files with 98 additions and 19 deletions

View File

@@ -19,6 +19,7 @@ from twitter_cli.client import (
from twitter_cli.exceptions import TwitterAPIError from twitter_cli.exceptions import TwitterAPIError
from twitter_cli.graphql import ( from twitter_cli.graphql import (
FEATURES, FEATURES,
FALLBACK_QUERY_IDS,
_build_graphql_url, _build_graphql_url,
_update_features_from_html, _update_features_from_html,
) )
@@ -204,6 +205,10 @@ class TestBuildGraphqlUrl:
) )
assert len(url) < 8000, f"URL too long: {len(url)} chars" assert len(url) < 8000, f"URL too long: {len(url)} chars"
def test_searchtimeline_fallback_query_id_regression(self):
"""Keep SearchTimeline fallback aligned with the live operation after issue #39."""
assert FALLBACK_QUERY_IDS["SearchTimeline"] == "rkp6b4vtR9u7v3naGoOzUQ"
# ── _best_chrome_target ────────────────────────────────────────────────── # ── _best_chrome_target ──────────────────────────────────────────────────
@@ -1175,3 +1180,68 @@ class TestCreateTweetWithMedia:
result = client.create_tweet("no media") result = client.create_tweet("no media")
assert result == "88" assert result == "88"
assert captured_body["media"]["media_entities"] == [] assert captured_body["media"]["media_entities"] == []
# ── fetch_search uses POST ────────────────────────────────────────────────
class TestFetchSearchUsesPost:
"""Verify that fetch_search routes through _graphql_post (not GET)."""
def _make_client(self):
client = TwitterClient.__new__(TwitterClient)
client._auth_token = "tok"
client._ct0 = "ct0"
client._cookie_string = None
client._request_delay = 0
client._max_retries = 0
client._retry_base_delay = 0
client._max_count = 200
client._client_transaction = None
client._ct_init_attempted = True
return client
def test_fetch_search_calls_graphql_post(self):
"""fetch_search must use POST, not GET, for SearchTimeline."""
client = self._make_client()
post_calls = []
get_calls = []
def mock_post(operation_name, variables, features=None):
post_calls.append((operation_name, variables))
return {"data": {"search_by_raw_query": {"search_timeline": {"timeline": {"instructions": []}}}}}
def mock_get(operation_name, variables, features, field_toggles=None): # pragma: no cover
get_calls.append(operation_name)
return {}
client._graphql_post = mock_post
client._graphql_get = mock_get
results = client.fetch_search("AI agent", count=5)
assert len(get_calls) == 0, "fetch_search must NOT call _graphql_get"
assert len(post_calls) == 1
op_name, variables = post_calls[0]
assert op_name == "SearchTimeline"
assert variables["rawQuery"] == "AI agent"
assert variables["product"] == "Top"
assert results == []
def test_fetch_search_passes_product_param(self):
"""fetch_search forwards the product parameter correctly."""
client = self._make_client()
captured = {}
def mock_post(operation_name, variables, features=None):
captured.update(variables)
return {"data": {"search_by_raw_query": {"search_timeline": {"timeline": {"instructions": []}}}}}
client._graphql_post = mock_post
client._graphql_get = lambda *a, **kw: {} # pragma: no cover
client.fetch_search("python", count=3, product="Latest")
assert captured.get("product") == "Latest"
assert captured.get("querySource") == "typed_query"

View File

@@ -341,6 +341,7 @@ class TwitterClient:
count: Max number of tweets to return. count: Max number of tweets to return.
product: Search tab — "Top", "Latest", "People", "Photos", "Videos". product: Search tab — "Top", "Latest", "People", "Photos", "Videos".
""" """
# Twitter migrated SearchTimeline from GET to POST — use _graphql_post.
return self._fetch_timeline( return self._fetch_timeline(
"SearchTimeline", "SearchTimeline",
count, count,
@@ -353,6 +354,7 @@ class TwitterClient:
"product": product, "product": product,
}, },
override_base_variables=True, override_base_variables=True,
use_post=True,
) )
def fetch_tweet_detail(self, tweet_id, count=20): def fetch_tweet_detail(self, tweet_id, count=20):
@@ -730,14 +732,16 @@ class TwitterClient:
# ── Internal: timeline / user list fetchers ────────────────────── # ── Internal: timeline / user list fetchers ──────────────────────
def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None): def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None, use_post=False):
# type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]]) -> List[Tweet] # type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]], bool) -> List[Tweet]
"""Generic timeline fetcher with pagination and deduplication. """Generic timeline fetcher with pagination and deduplication.
Args: Args:
override_base_variables: If True, use only extra_variables + count/cursor override_base_variables: If True, use only extra_variables + count/cursor
instead of the default timeline base variables. Needed for instead of the default timeline base variables. Needed for
endpoints like SearchTimeline that reject unknown variables. endpoints like SearchTimeline that reject unknown variables.
use_post: If True, send request via POST instead of GET. Required for
endpoints like SearchTimeline that Twitter migrated to POST.
""" """
if count <= 0: if count <= 0:
return [] return []
@@ -768,7 +772,10 @@ class TwitterClient:
if cursor: if cursor:
variables["cursor"] = cursor variables["cursor"] = cursor
data = self._graphql_get(operation_name, variables, FEATURES, field_toggles=field_toggles) if use_post:
data = self._graphql_post(operation_name, variables, FEATURES)
else:
data = self._graphql_get(operation_name, variables, FEATURES, field_toggles=field_toggles)
new_tweets, next_cursor = parse_timeline_response(data, get_instructions) new_tweets, next_cursor = parse_timeline_response(data, get_instructions)
for tweet in new_tweets: for tweet in new_tweets:
@@ -1075,6 +1082,8 @@ class TwitterClient:
) )
home_page_response = bs4.BeautifulSoup(home_page.content, "html.parser") home_page_response = bs4.BeautifulSoup(home_page.content, "html.parser")
ondemand_url = get_ondemand_file_url(response=home_page_response) ondemand_url = get_ondemand_file_url(response=home_page_response)
if not ondemand_url:
raise ValueError("Failed to extract ondemand file URL from homepage")
ondemand_file = cffi_session.get( ondemand_file = cffi_session.get(
ondemand_url, headers=ct_headers, timeout=10, ondemand_url, headers=ct_headers, timeout=10,
) )

View File

@@ -27,26 +27,26 @@ TWITTER_OPENAPI_URL = (
# ── Fallback (hardcoded) queryIds ──────────────────────────────────────── # ── Fallback (hardcoded) queryIds ────────────────────────────────────────
FALLBACK_QUERY_IDS = { FALLBACK_QUERY_IDS = {
"HomeTimeline": "HCosKfLNW1AcOo3la3mMgg", "HomeTimeline": "L8Lb9oomccM012S7fQ-QKA",
"HomeLatestTimeline": "U0cdisy7QFIoTfu3-Okw0A", "HomeLatestTimeline": "tzmrSIWxyV4IRRh9nij6TQ",
"UserByScreenName": "qRednkZG-rn1P6b48NINmQ", "UserByScreenName": "IGgvgiOx4QZndDHuD3x9TQ",
"UserTweets": "E3opETHurmVJflFsUBVuUQ", "UserTweets": "O0epvwaQPUx-bT9YlqlL6w",
"TweetDetail": "nBS-WpgA6ZG0CyNHD517JQ", "TweetDetail": "xIYgDwjboktoFeXe_fgacw",
"Likes": "dv5-II7_Bup_PHish7p6fw", "Likes": "RozQdCp4CilQzrcuU0NY5w",
"SearchTimeline": "MJpyQGqgklrVl_0X9gNy3A", "SearchTimeline": "rkp6b4vtR9u7v3naGoOzUQ",
"Bookmarks": "uzboyXSHSJrR-mGJqep0TQ", "Bookmarks": "uzboyXSHSJrR-mGJqep0TQ",
"ListLatestTweetsTimeline": "ZBbXrl0FVnTqp7K6EAADog", "ListLatestTweetsTimeline": "fb_6wmHD2dk9D-xYXOQlgw",
"Followers": "IOh4aS6UdGWGJUYTqliQ7Q", "Followers": "Enf9DNUZYiT037aersI5gg",
"Following": "zx6e-TLzRkeDO_a7p4b3JQ", "Following": "ntIPnH1WMBKW--4Tn1q71A",
"CreateTweet": "bDE2rBtZb3uyrczSZ_pI9g", "CreateTweet": "zkcFc6F-RKRgWN8HUkJfZg",
"DeleteTweet": "VaenaVgh5q5ih7kvyVjgtg", "DeleteTweet": "nxpZCY2K-I6QoFHAHeojFQ",
"FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A", "FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A",
"UnfavoriteTweet": "ZYKSe-w7KEslx3JhSIk5LA", "UnfavoriteTweet": "ZYKSe-w7KEslx3JhSIk5LA",
"CreateRetweet": "ojPdsZsimiJrUGLR1sjVsA", "CreateRetweet": "mbRO74GrOvSfRcJnlMapnQ",
"DeleteRetweet": "iQtK4dl5hBmXewYZuEOKVw", "DeleteRetweet": "ZyZigVsNiFO6v1dEks1eWg",
"CreateBookmark": "aoDbu3RHznuiSkQ9aNM67Q", "CreateBookmark": "aoDbu3RHznuiSkQ9aNM67Q",
"DeleteBookmark": "Wlmlj2-xISYCixDmuS8KNg", "DeleteBookmark": "Wlmlj2-xzyS1GN3a6cj-mQ",
"TweetResultByRestId": "7xflPyRiUxGVbJd4uWmbfg", "TweetResultByRestId": "zy39CwTyYhU-_0LP7dljjg",
"BookmarkFoldersSlice": "i78YDd0Tza-dV4SYs58kRg", "BookmarkFoldersSlice": "i78YDd0Tza-dV4SYs58kRg",
"BookmarkFolderTimeline": "hNY7X2xE2N7HVF6Qb_mu6w", "BookmarkFolderTimeline": "hNY7X2xE2N7HVF6Qb_mu6w",
} }