fix: SearchTimeline POST + ondemand URL guard + refresh queryIds
Twitter migrated SearchTimeline from GET to POST; update fetch_search to route through _graphql_post via a new use_post flag on _fetch_timeline, avoiding duplicating the pagination logic (closes #39, refs #40, #42). Also guard against get_ondemand_file_url returning None so the error message is clear instead of crashing on NoneType.split (closes #43-adjacent). Refresh all FALLBACK_QUERY_IDS from live JS bundles — 18 operations updated.
This commit is contained in:
@@ -19,6 +19,7 @@ from twitter_cli.client import (
|
|||||||
from twitter_cli.exceptions import TwitterAPIError
|
from twitter_cli.exceptions import TwitterAPIError
|
||||||
from twitter_cli.graphql import (
|
from twitter_cli.graphql import (
|
||||||
FEATURES,
|
FEATURES,
|
||||||
|
FALLBACK_QUERY_IDS,
|
||||||
_build_graphql_url,
|
_build_graphql_url,
|
||||||
_update_features_from_html,
|
_update_features_from_html,
|
||||||
)
|
)
|
||||||
@@ -204,6 +205,10 @@ class TestBuildGraphqlUrl:
|
|||||||
)
|
)
|
||||||
assert len(url) < 8000, f"URL too long: {len(url)} chars"
|
assert len(url) < 8000, f"URL too long: {len(url)} chars"
|
||||||
|
|
||||||
|
def test_searchtimeline_fallback_query_id_regression(self):
|
||||||
|
"""Keep SearchTimeline fallback aligned with the live operation after issue #39."""
|
||||||
|
assert FALLBACK_QUERY_IDS["SearchTimeline"] == "rkp6b4vtR9u7v3naGoOzUQ"
|
||||||
|
|
||||||
|
|
||||||
# ── _best_chrome_target ──────────────────────────────────────────────────
|
# ── _best_chrome_target ──────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -1175,3 +1180,68 @@ class TestCreateTweetWithMedia:
|
|||||||
result = client.create_tweet("no media")
|
result = client.create_tweet("no media")
|
||||||
assert result == "88"
|
assert result == "88"
|
||||||
assert captured_body["media"]["media_entities"] == []
|
assert captured_body["media"]["media_entities"] == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── fetch_search uses POST ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
class TestFetchSearchUsesPost:
|
||||||
|
"""Verify that fetch_search routes through _graphql_post (not GET)."""
|
||||||
|
|
||||||
|
def _make_client(self):
|
||||||
|
client = TwitterClient.__new__(TwitterClient)
|
||||||
|
client._auth_token = "tok"
|
||||||
|
client._ct0 = "ct0"
|
||||||
|
client._cookie_string = None
|
||||||
|
client._request_delay = 0
|
||||||
|
client._max_retries = 0
|
||||||
|
client._retry_base_delay = 0
|
||||||
|
client._max_count = 200
|
||||||
|
client._client_transaction = None
|
||||||
|
client._ct_init_attempted = True
|
||||||
|
return client
|
||||||
|
|
||||||
|
def test_fetch_search_calls_graphql_post(self):
|
||||||
|
"""fetch_search must use POST, not GET, for SearchTimeline."""
|
||||||
|
client = self._make_client()
|
||||||
|
|
||||||
|
post_calls = []
|
||||||
|
get_calls = []
|
||||||
|
|
||||||
|
def mock_post(operation_name, variables, features=None):
|
||||||
|
post_calls.append((operation_name, variables))
|
||||||
|
return {"data": {"search_by_raw_query": {"search_timeline": {"timeline": {"instructions": []}}}}}
|
||||||
|
|
||||||
|
def mock_get(operation_name, variables, features, field_toggles=None): # pragma: no cover
|
||||||
|
get_calls.append(operation_name)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
client._graphql_post = mock_post
|
||||||
|
client._graphql_get = mock_get
|
||||||
|
|
||||||
|
results = client.fetch_search("AI agent", count=5)
|
||||||
|
|
||||||
|
assert len(get_calls) == 0, "fetch_search must NOT call _graphql_get"
|
||||||
|
assert len(post_calls) == 1
|
||||||
|
op_name, variables = post_calls[0]
|
||||||
|
assert op_name == "SearchTimeline"
|
||||||
|
assert variables["rawQuery"] == "AI agent"
|
||||||
|
assert variables["product"] == "Top"
|
||||||
|
assert results == []
|
||||||
|
|
||||||
|
def test_fetch_search_passes_product_param(self):
|
||||||
|
"""fetch_search forwards the product parameter correctly."""
|
||||||
|
client = self._make_client()
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def mock_post(operation_name, variables, features=None):
|
||||||
|
captured.update(variables)
|
||||||
|
return {"data": {"search_by_raw_query": {"search_timeline": {"timeline": {"instructions": []}}}}}
|
||||||
|
|
||||||
|
client._graphql_post = mock_post
|
||||||
|
client._graphql_get = lambda *a, **kw: {} # pragma: no cover
|
||||||
|
|
||||||
|
client.fetch_search("python", count=3, product="Latest")
|
||||||
|
|
||||||
|
assert captured.get("product") == "Latest"
|
||||||
|
assert captured.get("querySource") == "typed_query"
|
||||||
|
|||||||
@@ -341,6 +341,7 @@ class TwitterClient:
|
|||||||
count: Max number of tweets to return.
|
count: Max number of tweets to return.
|
||||||
product: Search tab — "Top", "Latest", "People", "Photos", "Videos".
|
product: Search tab — "Top", "Latest", "People", "Photos", "Videos".
|
||||||
"""
|
"""
|
||||||
|
# Twitter migrated SearchTimeline from GET to POST — use _graphql_post.
|
||||||
return self._fetch_timeline(
|
return self._fetch_timeline(
|
||||||
"SearchTimeline",
|
"SearchTimeline",
|
||||||
count,
|
count,
|
||||||
@@ -353,6 +354,7 @@ class TwitterClient:
|
|||||||
"product": product,
|
"product": product,
|
||||||
},
|
},
|
||||||
override_base_variables=True,
|
override_base_variables=True,
|
||||||
|
use_post=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
def fetch_tweet_detail(self, tweet_id, count=20):
|
def fetch_tweet_detail(self, tweet_id, count=20):
|
||||||
@@ -730,14 +732,16 @@ class TwitterClient:
|
|||||||
|
|
||||||
# ── Internal: timeline / user list fetchers ──────────────────────
|
# ── Internal: timeline / user list fetchers ──────────────────────
|
||||||
|
|
||||||
def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None):
|
def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None, use_post=False):
|
||||||
# type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]]) -> List[Tweet]
|
# type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]], bool) -> List[Tweet]
|
||||||
"""Generic timeline fetcher with pagination and deduplication.
|
"""Generic timeline fetcher with pagination and deduplication.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
override_base_variables: If True, use only extra_variables + count/cursor
|
override_base_variables: If True, use only extra_variables + count/cursor
|
||||||
instead of the default timeline base variables. Needed for
|
instead of the default timeline base variables. Needed for
|
||||||
endpoints like SearchTimeline that reject unknown variables.
|
endpoints like SearchTimeline that reject unknown variables.
|
||||||
|
use_post: If True, send request via POST instead of GET. Required for
|
||||||
|
endpoints like SearchTimeline that Twitter migrated to POST.
|
||||||
"""
|
"""
|
||||||
if count <= 0:
|
if count <= 0:
|
||||||
return []
|
return []
|
||||||
@@ -768,7 +772,10 @@ class TwitterClient:
|
|||||||
if cursor:
|
if cursor:
|
||||||
variables["cursor"] = cursor
|
variables["cursor"] = cursor
|
||||||
|
|
||||||
data = self._graphql_get(operation_name, variables, FEATURES, field_toggles=field_toggles)
|
if use_post:
|
||||||
|
data = self._graphql_post(operation_name, variables, FEATURES)
|
||||||
|
else:
|
||||||
|
data = self._graphql_get(operation_name, variables, FEATURES, field_toggles=field_toggles)
|
||||||
new_tweets, next_cursor = parse_timeline_response(data, get_instructions)
|
new_tweets, next_cursor = parse_timeline_response(data, get_instructions)
|
||||||
|
|
||||||
for tweet in new_tweets:
|
for tweet in new_tweets:
|
||||||
@@ -1075,6 +1082,8 @@ class TwitterClient:
|
|||||||
)
|
)
|
||||||
home_page_response = bs4.BeautifulSoup(home_page.content, "html.parser")
|
home_page_response = bs4.BeautifulSoup(home_page.content, "html.parser")
|
||||||
ondemand_url = get_ondemand_file_url(response=home_page_response)
|
ondemand_url = get_ondemand_file_url(response=home_page_response)
|
||||||
|
if not ondemand_url:
|
||||||
|
raise ValueError("Failed to extract ondemand file URL from homepage")
|
||||||
ondemand_file = cffi_session.get(
|
ondemand_file = cffi_session.get(
|
||||||
ondemand_url, headers=ct_headers, timeout=10,
|
ondemand_url, headers=ct_headers, timeout=10,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -27,26 +27,26 @@ TWITTER_OPENAPI_URL = (
|
|||||||
|
|
||||||
# ── Fallback (hardcoded) queryIds ────────────────────────────────────────
|
# ── Fallback (hardcoded) queryIds ────────────────────────────────────────
|
||||||
FALLBACK_QUERY_IDS = {
|
FALLBACK_QUERY_IDS = {
|
||||||
"HomeTimeline": "HCosKfLNW1AcOo3la3mMgg",
|
"HomeTimeline": "L8Lb9oomccM012S7fQ-QKA",
|
||||||
"HomeLatestTimeline": "U0cdisy7QFIoTfu3-Okw0A",
|
"HomeLatestTimeline": "tzmrSIWxyV4IRRh9nij6TQ",
|
||||||
"UserByScreenName": "qRednkZG-rn1P6b48NINmQ",
|
"UserByScreenName": "IGgvgiOx4QZndDHuD3x9TQ",
|
||||||
"UserTweets": "E3opETHurmVJflFsUBVuUQ",
|
"UserTweets": "O0epvwaQPUx-bT9YlqlL6w",
|
||||||
"TweetDetail": "nBS-WpgA6ZG0CyNHD517JQ",
|
"TweetDetail": "xIYgDwjboktoFeXe_fgacw",
|
||||||
"Likes": "dv5-II7_Bup_PHish7p6fw",
|
"Likes": "RozQdCp4CilQzrcuU0NY5w",
|
||||||
"SearchTimeline": "MJpyQGqgklrVl_0X9gNy3A",
|
"SearchTimeline": "rkp6b4vtR9u7v3naGoOzUQ",
|
||||||
"Bookmarks": "uzboyXSHSJrR-mGJqep0TQ",
|
"Bookmarks": "uzboyXSHSJrR-mGJqep0TQ",
|
||||||
"ListLatestTweetsTimeline": "ZBbXrl0FVnTqp7K6EAADog",
|
"ListLatestTweetsTimeline": "fb_6wmHD2dk9D-xYXOQlgw",
|
||||||
"Followers": "IOh4aS6UdGWGJUYTqliQ7Q",
|
"Followers": "Enf9DNUZYiT037aersI5gg",
|
||||||
"Following": "zx6e-TLzRkeDO_a7p4b3JQ",
|
"Following": "ntIPnH1WMBKW--4Tn1q71A",
|
||||||
"CreateTweet": "bDE2rBtZb3uyrczSZ_pI9g",
|
"CreateTweet": "zkcFc6F-RKRgWN8HUkJfZg",
|
||||||
"DeleteTweet": "VaenaVgh5q5ih7kvyVjgtg",
|
"DeleteTweet": "nxpZCY2K-I6QoFHAHeojFQ",
|
||||||
"FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A",
|
"FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A",
|
||||||
"UnfavoriteTweet": "ZYKSe-w7KEslx3JhSIk5LA",
|
"UnfavoriteTweet": "ZYKSe-w7KEslx3JhSIk5LA",
|
||||||
"CreateRetweet": "ojPdsZsimiJrUGLR1sjVsA",
|
"CreateRetweet": "mbRO74GrOvSfRcJnlMapnQ",
|
||||||
"DeleteRetweet": "iQtK4dl5hBmXewYZuEOKVw",
|
"DeleteRetweet": "ZyZigVsNiFO6v1dEks1eWg",
|
||||||
"CreateBookmark": "aoDbu3RHznuiSkQ9aNM67Q",
|
"CreateBookmark": "aoDbu3RHznuiSkQ9aNM67Q",
|
||||||
"DeleteBookmark": "Wlmlj2-xISYCixDmuS8KNg",
|
"DeleteBookmark": "Wlmlj2-xzyS1GN3a6cj-mQ",
|
||||||
"TweetResultByRestId": "7xflPyRiUxGVbJd4uWmbfg",
|
"TweetResultByRestId": "zy39CwTyYhU-_0LP7dljjg",
|
||||||
"BookmarkFoldersSlice": "i78YDd0Tza-dV4SYs58kRg",
|
"BookmarkFoldersSlice": "i78YDd0Tza-dV4SYs58kRg",
|
||||||
"BookmarkFolderTimeline": "hNY7X2xE2N7HVF6Qb_mu6w",
|
"BookmarkFolderTimeline": "hNY7X2xE2N7HVF6Qb_mu6w",
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user