From 199a1490f9a2c1289970ef872944e1d3073e1e72 Mon Sep 17 00:00:00 2001 From: jackwener Date: Fri, 20 Mar 2026 18:21:29 +0800 Subject: [PATCH] fix: SearchTimeline POST + ondemand URL guard + refresh queryIds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twitter migrated SearchTimeline from GET to POST; update fetch_search to route through _graphql_post via a new use_post flag on _fetch_timeline, avoiding duplicating the pagination logic (closes #39, refs #40, #42). Also guard against get_ondemand_file_url returning None so the error message is clear instead of crashing on NoneType.split (closes #43-adjacent). Refresh all FALLBACK_QUERY_IDS from live JS bundles — 18 operations updated. --- tests/test_client.py | 70 ++++++++++++++++++++++++++++++++++++++++++ twitter_cli/client.py | 15 +++++++-- twitter_cli/graphql.py | 32 +++++++++---------- 3 files changed, 98 insertions(+), 19 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 8407c45..cb92a14 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -19,6 +19,7 @@ from twitter_cli.client import ( from twitter_cli.exceptions import TwitterAPIError from twitter_cli.graphql import ( FEATURES, + FALLBACK_QUERY_IDS, _build_graphql_url, _update_features_from_html, ) @@ -204,6 +205,10 @@ class TestBuildGraphqlUrl: ) assert len(url) < 8000, f"URL too long: {len(url)} chars" + def test_searchtimeline_fallback_query_id_regression(self): + """Keep SearchTimeline fallback aligned with the live operation after issue #39.""" + assert FALLBACK_QUERY_IDS["SearchTimeline"] == "rkp6b4vtR9u7v3naGoOzUQ" + # ── _best_chrome_target ────────────────────────────────────────────────── @@ -1175,3 +1180,68 @@ class TestCreateTweetWithMedia: result = client.create_tweet("no media") assert result == "88" assert captured_body["media"]["media_entities"] == [] + + +# ── fetch_search uses POST ──────────────────────────────────────────────── + +class TestFetchSearchUsesPost: + """Verify that fetch_search routes through _graphql_post (not GET).""" + + def _make_client(self): + client = TwitterClient.__new__(TwitterClient) + client._auth_token = "tok" + client._ct0 = "ct0" + client._cookie_string = None + client._request_delay = 0 + client._max_retries = 0 + client._retry_base_delay = 0 + client._max_count = 200 + client._client_transaction = None + client._ct_init_attempted = True + return client + + def test_fetch_search_calls_graphql_post(self): + """fetch_search must use POST, not GET, for SearchTimeline.""" + client = self._make_client() + + post_calls = [] + get_calls = [] + + def mock_post(operation_name, variables, features=None): + post_calls.append((operation_name, variables)) + return {"data": {"search_by_raw_query": {"search_timeline": {"timeline": {"instructions": []}}}}} + + def mock_get(operation_name, variables, features, field_toggles=None): # pragma: no cover + get_calls.append(operation_name) + return {} + + client._graphql_post = mock_post + client._graphql_get = mock_get + + results = client.fetch_search("AI agent", count=5) + + assert len(get_calls) == 0, "fetch_search must NOT call _graphql_get" + assert len(post_calls) == 1 + op_name, variables = post_calls[0] + assert op_name == "SearchTimeline" + assert variables["rawQuery"] == "AI agent" + assert variables["product"] == "Top" + assert results == [] + + def test_fetch_search_passes_product_param(self): + """fetch_search forwards the product parameter correctly.""" + client = self._make_client() + + captured = {} + + def mock_post(operation_name, variables, features=None): + captured.update(variables) + return {"data": {"search_by_raw_query": {"search_timeline": {"timeline": {"instructions": []}}}}} + + client._graphql_post = mock_post + client._graphql_get = lambda *a, **kw: {} # pragma: no cover + + client.fetch_search("python", count=3, product="Latest") + + assert captured.get("product") == "Latest" + assert captured.get("querySource") == "typed_query" diff --git a/twitter_cli/client.py b/twitter_cli/client.py index 6e7d707..9ef4f5a 100644 --- a/twitter_cli/client.py +++ b/twitter_cli/client.py @@ -341,6 +341,7 @@ class TwitterClient: count: Max number of tweets to return. product: Search tab — "Top", "Latest", "People", "Photos", "Videos". """ + # Twitter migrated SearchTimeline from GET to POST — use _graphql_post. return self._fetch_timeline( "SearchTimeline", count, @@ -353,6 +354,7 @@ class TwitterClient: "product": product, }, override_base_variables=True, + use_post=True, ) def fetch_tweet_detail(self, tweet_id, count=20): @@ -730,14 +732,16 @@ class TwitterClient: # ── Internal: timeline / user list fetchers ────────────────────── - def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None): - # type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]]) -> List[Tweet] + def _fetch_timeline(self, operation_name, count, get_instructions, extra_variables=None, override_base_variables=False, field_toggles=None, use_post=False): + # type: (str, int, Callable[[Any], Any], Optional[Dict[str, Any]], bool, Optional[Dict[str, Any]], bool) -> List[Tweet] """Generic timeline fetcher with pagination and deduplication. Args: override_base_variables: If True, use only extra_variables + count/cursor instead of the default timeline base variables. Needed for endpoints like SearchTimeline that reject unknown variables. + use_post: If True, send request via POST instead of GET. Required for + endpoints like SearchTimeline that Twitter migrated to POST. """ if count <= 0: return [] @@ -768,7 +772,10 @@ class TwitterClient: if cursor: variables["cursor"] = cursor - data = self._graphql_get(operation_name, variables, FEATURES, field_toggles=field_toggles) + if use_post: + data = self._graphql_post(operation_name, variables, FEATURES) + else: + data = self._graphql_get(operation_name, variables, FEATURES, field_toggles=field_toggles) new_tweets, next_cursor = parse_timeline_response(data, get_instructions) for tweet in new_tweets: @@ -1075,6 +1082,8 @@ class TwitterClient: ) home_page_response = bs4.BeautifulSoup(home_page.content, "html.parser") ondemand_url = get_ondemand_file_url(response=home_page_response) + if not ondemand_url: + raise ValueError("Failed to extract ondemand file URL from homepage") ondemand_file = cffi_session.get( ondemand_url, headers=ct_headers, timeout=10, ) diff --git a/twitter_cli/graphql.py b/twitter_cli/graphql.py index 40e96bb..d600497 100644 --- a/twitter_cli/graphql.py +++ b/twitter_cli/graphql.py @@ -27,26 +27,26 @@ TWITTER_OPENAPI_URL = ( # ── Fallback (hardcoded) queryIds ──────────────────────────────────────── FALLBACK_QUERY_IDS = { - "HomeTimeline": "HCosKfLNW1AcOo3la3mMgg", - "HomeLatestTimeline": "U0cdisy7QFIoTfu3-Okw0A", - "UserByScreenName": "qRednkZG-rn1P6b48NINmQ", - "UserTweets": "E3opETHurmVJflFsUBVuUQ", - "TweetDetail": "nBS-WpgA6ZG0CyNHD517JQ", - "Likes": "dv5-II7_Bup_PHish7p6fw", - "SearchTimeline": "MJpyQGqgklrVl_0X9gNy3A", + "HomeTimeline": "L8Lb9oomccM012S7fQ-QKA", + "HomeLatestTimeline": "tzmrSIWxyV4IRRh9nij6TQ", + "UserByScreenName": "IGgvgiOx4QZndDHuD3x9TQ", + "UserTweets": "O0epvwaQPUx-bT9YlqlL6w", + "TweetDetail": "xIYgDwjboktoFeXe_fgacw", + "Likes": "RozQdCp4CilQzrcuU0NY5w", + "SearchTimeline": "rkp6b4vtR9u7v3naGoOzUQ", "Bookmarks": "uzboyXSHSJrR-mGJqep0TQ", - "ListLatestTweetsTimeline": "ZBbXrl0FVnTqp7K6EAADog", - "Followers": "IOh4aS6UdGWGJUYTqliQ7Q", - "Following": "zx6e-TLzRkeDO_a7p4b3JQ", - "CreateTweet": "bDE2rBtZb3uyrczSZ_pI9g", - "DeleteTweet": "VaenaVgh5q5ih7kvyVjgtg", + "ListLatestTweetsTimeline": "fb_6wmHD2dk9D-xYXOQlgw", + "Followers": "Enf9DNUZYiT037aersI5gg", + "Following": "ntIPnH1WMBKW--4Tn1q71A", + "CreateTweet": "zkcFc6F-RKRgWN8HUkJfZg", + "DeleteTweet": "nxpZCY2K-I6QoFHAHeojFQ", "FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A", "UnfavoriteTweet": "ZYKSe-w7KEslx3JhSIk5LA", - "CreateRetweet": "ojPdsZsimiJrUGLR1sjVsA", - "DeleteRetweet": "iQtK4dl5hBmXewYZuEOKVw", + "CreateRetweet": "mbRO74GrOvSfRcJnlMapnQ", + "DeleteRetweet": "ZyZigVsNiFO6v1dEks1eWg", "CreateBookmark": "aoDbu3RHznuiSkQ9aNM67Q", - "DeleteBookmark": "Wlmlj2-xISYCixDmuS8KNg", - "TweetResultByRestId": "7xflPyRiUxGVbJd4uWmbfg", + "DeleteBookmark": "Wlmlj2-xzyS1GN3a6cj-mQ", + "TweetResultByRestId": "zy39CwTyYhU-_0LP7dljjg", "BookmarkFoldersSlice": "i78YDd0Tza-dV4SYs58kRg", "BookmarkFolderTimeline": "hNY7X2xE2N7HVF6Qb_mu6w", }