From a64e2286d592b907eba6706ba0a8615b8161c4f1 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 21 Apr 2023 17:41:48 +0200 Subject: [PATCH 01/14] Update badges --- .github/workflows/run-tests.yml | 2 +- README.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index d6315b3..ca93f30 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,4 +1,4 @@ -name: Run tests +name: Tests on: push: diff --git a/README.md b/README.md index 5ca2ee6..4f8235d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Nitter -[![Test Matrix](https://github.com/zedeus/nitter/workflows/CI/CD/badge.svg)](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD) +[![Test Matrix](https://github.com/zedeus/nitter/workflows/Tests/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/run-tests.yml) +[![Test Matrix](https://github.com/zedeus/nitter/workflows/Docker/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml) [![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license) A free and open source alternative Twitter front-end focused on privacy and From b67f3062fed36f78a827be4b31340ca79a6da4cf Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 21 Apr 2023 23:33:52 +0200 Subject: [PATCH 02/14] Fix card tests --- tests/test_card.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_card.py b/tests/test_card.py index da6ffde..78fe79c 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -76,7 +76,7 @@ class CardTest(BaseTestCase): c = Card(Conversation.main + " ") self.assert_text(title, c.title) self.assert_text(destination, c.destination) - self.assertIn('_img', self.get_image_url(c.image + ' img')) + self.assertIn('/pic/', self.get_image_url(c.image + ' img')) if len(description) > 0: self.assert_text(description, c.description) if large: @@ -99,7 +99,7 @@ class CardTest(BaseTestCase): c = Card(Conversation.main + " ") self.assert_text(title, c.title) self.assert_text(destination, c.destination) - self.assertIn('_img', self.get_image_url(c.image + ' img')) + self.assertIn('/pic/', self.get_image_url(c.image + ' img')) self.assert_element_visible('.card-overlay') if len(description) > 0: self.assert_text(description, c.description) From fe22a45c7e3221f9abd7106b304f1309c904b967 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 21 Apr 2023 23:43:46 +0200 Subject: [PATCH 03/14] Make Docker workflow depend on tests --- .github/workflows/build-docker.yml | 4 ++++ .github/workflows/run-tests.yml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 754f080..1250195 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -8,7 +8,10 @@ on: - master jobs: + tests: + uses: ./.github/workflows/run-tests.yml build-docker-amd64: + depends: [tests] runs-on: buildjet-2vcpu-ubuntu-2204 steps: - uses: actions/checkout@v3 @@ -33,6 +36,7 @@ jobs: push: true tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }} build-docker-arm64: + depends: [tests] runs-on: buildjet-2vcpu-ubuntu-2204-arm steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ca93f30..140b6bf 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,6 +4,9 @@ on: push: paths-ignore: - "*.md" + branches-ignore: + - master + workflow_call: jobs: test: From 2254a0728c587ebcec51ff08da0bf145606a629e Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 21 Apr 2023 23:52:14 +0200 Subject: [PATCH 04/14] depends -> needs --- .github/workflows/build-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 1250195..765e7a0 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -11,7 +11,7 @@ jobs: tests: uses: ./.github/workflows/run-tests.yml build-docker-amd64: - depends: [tests] + needs: [tests] runs-on: buildjet-2vcpu-ubuntu-2204 steps: - uses: actions/checkout@v3 @@ -36,7 +36,7 @@ jobs: push: true tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }} build-docker-arm64: - depends: [tests] + needs: [tests] runs-on: buildjet-2vcpu-ubuntu-2204-arm steps: - uses: actions/checkout@v3 From 774063fb5a46633a1984900734b7300ecb1bfb4f Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 20 May 2023 00:52:55 +0200 Subject: [PATCH 05/14] Fix pinned "TweetPreviewDisplay" crash --- src/parser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser.nim b/src/parser.nim index fe2fe5b..78f9bca 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -384,7 +384,7 @@ proc parseGraphTweet(js: JsonNode): Tweet = return Tweet(available: false) case js{"__typename"}.getStr - of "TweetUnavailable": + of "TweetUnavailable", "TweetPreviewDisplay": return Tweet(available: false) of "TweetTombstone": return Tweet( From b516ec367b41fad1185e3e408230cd61e24e2ea2 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 20 May 2023 00:59:34 +0200 Subject: [PATCH 06/14] Remove outdated tweet from card test --- tests/test_card.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_card.py b/tests/test_card.py index 78fe79c..696b9d5 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -35,11 +35,6 @@ no_thumb = [ 'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.', 'github.com'], - ['mobile_test/status/490378953744318464', - 'Nantasket Beach', - 'Explore this photo titled Nantasket Beach by Ben Sandofsky (@sandofsky) on 500px', - '500px.com'], - ['nim_lang/status/1082989146040340480', 'Nim in 2018: A short recap', 'Posted by u/miran1 - 36 votes and 46 comments', From e3b3b38a2d43a83b5fc2239ab41e864ee686fb2f Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 20 May 2023 02:10:37 +0200 Subject: [PATCH 07/14] Add proper tombstone for subscriber tweets --- src/consts.nim | 18 +++++++++++------- src/parser.nim | 13 ++++++------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/consts.nim b/src/consts.nim index 27e82f9..f22581f 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -11,13 +11,13 @@ const userSearch* = api / "1.1/users/search.json" graphql = api / "graphql" - graphUser* = graphql / "8mPfHBetXOg-EHAyeVxUoA/UserByScreenName" - graphUserById* = graphql / "nI8WydSd-X-lQIVo6bdktQ/UserByRestId" - graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets" - graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies" - graphUserMedia* = graphql / "MA_EP2a21zpzNWKRkaPBMg/UserMedia" - graphTweet* = graphql / "6I7Hm635Q6ftv69L8VrSeQ/TweetDetail" - graphTweetResult* = graphql / "rt-rHeSJ-2H9O9gxWQcPcg/TweetResultByRestId" + graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName" + graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId" + graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets" + graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies" + graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia" + graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail" + graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId" graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline" graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId" graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug" @@ -50,10 +50,13 @@ const gqlFeatures* = """{ "blue_business_profile_image_shape_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": false, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false, + "highlights_tweets_tab_ui_enabled": false, "interactive_text_enabled": false, "longform_notetweets_consumption_enabled": true, + "longform_notetweets_inline_media_enabled": false, "longform_notetweets_richtext_consumption_enabled": true, "longform_notetweets_rich_text_read_enabled": false, "responsive_web_edit_tweet_api_enabled": false, @@ -63,6 +66,7 @@ const "responsive_web_graphql_timeline_navigation_enabled": false, "responsive_web_text_conversations_enabled": false, "responsive_web_twitter_blue_verified_badge_is_enabled": true, + "rweb_lists_timeline_redesign_enabled": true, "spaces_2022_h2_clipping": true, "spaces_2022_h2_spaces_communities": true, "standardized_nudges_misinfo": false, diff --git a/src/parser.nim b/src/parser.nim index 78f9bca..5ec21e4 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -381,16 +381,15 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail = proc parseGraphTweet(js: JsonNode): Tweet = if js.kind == JNull: - return Tweet(available: false) + return Tweet() case js{"__typename"}.getStr - of "TweetUnavailable", "TweetPreviewDisplay": - return Tweet(available: false) + of "TweetUnavailable": + return Tweet() of "TweetTombstone": - return Tweet( - available: false, - text: js{"tombstone", "text"}.getTombstone - ) + return Tweet(text: js{"tombstone", "text"}.getTombstone) + of "TweetPreviewDisplay": + return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.") of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}) From f7e878c1260acc87fa53331bd5dc4584791baefd Mon Sep 17 00:00:00 2001 From: PrivacyDevel <105459436+PrivacyDevel@users.noreply.github.com> Date: Tue, 30 May 2023 11:37:35 +0000 Subject: [PATCH 08/14] fixed bug that caused threads on user profiles to be hidden (#885) --- src/parser.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/parser.nim b/src/parser.nim index 5ec21e4..5b0d584 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -490,6 +490,10 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline = if not tweet.available: tweet.id = parseBiggestInt(entryId.getId()) result.content.add tweet + elif entryId.startsWith("profile-conversation") or entryId.startsWith("homeConversation"): + let (thread, self) = parseGraphThread(e) + for tweet in thread.content: + result.content.add tweet elif entryId.startsWith("cursor-bottom"): result.bottom = e{"content", "value"}.getStr From 38985af6ed30f050201b15425cdac0dc2e286b6d Mon Sep 17 00:00:00 2001 From: PrivacyDevel <105459436+PrivacyDevel@users.noreply.github.com> Date: Tue, 30 May 2023 21:42:14 +0000 Subject: [PATCH 09/14] fixed bug that caused everybody to be displayed as verified (#890) --- src/parser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser.nim b/src/parser.nim index 5b0d584..38dbb24 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -33,7 +33,7 @@ proc parseGraphUser(js: JsonNode): User = result = parseUser(user{"legacy"}) if "is_blue_verified" in user: - result.verified = true + result.verified = user{"is_blue_verified"}.getBool() proc parseGraphList*(js: JsonNode): List = if js.isNull: return From dcf73354ff173c0407c62f84ea3bb90a130303c1 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 1 Jul 2023 22:07:37 +0200 Subject: [PATCH 10/14] Fix GraphQL user crash with invalid JSON --- src/experimental/parser/graphql.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index 36014e3..0f08c4f 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -4,6 +4,9 @@ import user, ../types/[graphuser, graphlistmembers] from ../../types import User, Result, Query, QueryKind proc parseGraphUser*(json: string): User = + if json.len == 0 or json[0] != '{': + return + let raw = json.fromJson(GraphUser) if raw.data.user.result.reason.get("") == "Suspended": From 0bc3c153d9b38a3c02f321fb64a375fef6b97e8e Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 10 Jul 2023 11:25:34 +0200 Subject: [PATCH 11/14] Fix everything (#927) * Switch bearer token and endpoints, update parser * Enable user search, disable tweet search * Disable multi-user timelines for now * Fix parsing of pinned tombstone --- src/api.nim | 26 ++++----- src/consts.nim | 67 +++++++++-------------- src/experimental/parser/graphql.nim | 8 +-- src/experimental/types/graphuser.nim | 4 +- src/parser.nim | 82 ++++++++++++++-------------- src/routes/embed.nim | 12 ++-- src/routes/rss.nim | 52 +++++++++--------- src/routes/search.nim | 14 +++-- src/routes/timeline.nim | 43 ++++++--------- src/sass/tweet/thread.scss | 26 +++++++++ src/tokens.nim | 9 ++- src/types.nim | 5 +- src/views/rss.nimf | 43 ++++++++------- src/views/search.nim | 2 +- src/views/timeline.nim | 48 ++++++++-------- src/views/tweet.nim | 18 +++--- tests/test_card.py | 35 +++--------- tests/test_profile.py | 4 +- tests/test_search.py | 10 ++-- tests/test_tweet.py | 16 +++--- 20 files changed, 260 insertions(+), 264 deletions(-) diff --git a/src/api.nim b/src/api.nim index b23aa87..d99eb3d 100644 --- a/src/api.nim +++ b/src/api.nim @@ -7,20 +7,20 @@ import experimental/parser as newParser proc getGraphUser*(username: string): Future[User] {.async.} = if username.len == 0: return let - variables = %*{"screen_name": username} - params = {"variables": $variables, "features": gqlFeatures} + variables = """{"screen_name": "$1"}""" % username + params = {"variables": variables, "features": gqlFeatures} js = await fetchRaw(graphUser ? params, Api.userScreenName) result = parseGraphUser(js) proc getGraphUserById*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return let - variables = %*{"userId": id} - params = {"variables": $variables, "features": gqlFeatures} + variables = """{"rest_id": "$1"}""" % id + params = {"variables": variables, "features": gqlFeatures} js = await fetchRaw(graphUserById ? params, Api.userRestId) result = parseGraphUser(js) -proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} = +proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" @@ -40,7 +40,7 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = variables = listTweetsVariables % [id, cursor] params = {"variables": variables, "features": gqlFeatures} js = await fetch(graphListTweets ? params, Api.listTweets) - result = parseGraphTimeline(js, "list", after) + result = parseGraphTimeline(js, "list", after).tweets proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = let @@ -50,8 +50,8 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = proc getGraphList*(id: string): Future[List] {.async.} = let - variables = %*{"listId": id} - params = {"variables": $variables, "features": gqlFeatures} + variables = """{"listId": "$1"}""" % id + params = {"variables": variables, "features": gqlFeatures} result = parseGraphList(await fetch(graphListById ? params, Api.list)) proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} = @@ -72,7 +72,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = if id.len == 0: return let - variables = tweetResultVariables % id + variables = """{"rest_id": "$1"}""" % id params = {"variables": variables, "features": gqlFeatures} js = await fetch(graphTweetResult ? params, Api.tweetResult) result = parseGraphTweetResult(js) @@ -95,10 +95,10 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} = if after.len > 0: result.replies = await getReplies(id, after) -proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} = +proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} = let q = genQueryParam(query) if q.len == 0 or q == emptyQuery: - return Result[Tweet](query: query, beginning: true) + return Profile(tweets: Timeline(query: query, beginning: true)) var variables = %*{ @@ -112,8 +112,8 @@ proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} = if after.len > 0: variables["cursor"] = % after let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphSearch(await fetch(url, Api.search), after) - result.query = query + result = Profile(tweets: parseGraphSearch(await fetch(url, Api.search), after)) + result.tweets.query = query proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = if query.text.len == 0: diff --git a/src/consts.nim b/src/consts.nim index f22581f..184f9da 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -2,7 +2,7 @@ import uri, sequtils, strutils const - auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" + auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" api = parseUri("https://api.twitter.com") activate* = $(api / "1.1/guest/activate.json") @@ -11,18 +11,18 @@ const userSearch* = api / "1.1/users/search.json" graphql = api / "graphql" - graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName" - graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId" - graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets" - graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies" - graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia" - graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail" - graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId" + graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" + graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery" + graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2" + graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2" + graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2" + graphTweet* = graphql / "83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2" + graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery" graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline" graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId" graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug" graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers" - graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline" + graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" timelineParams* = { "include_profile_interstitial_type": "0", @@ -49,10 +49,13 @@ const }.toSeq gqlFeatures* = """{ + "android_graphql_skip_api_media_color_palette": false, "blue_business_profile_image_shape_enabled": false, + "creator_subscriptions_subscription_count_enabled": false, "creator_subscriptions_tweet_preview_api_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": false, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false, + "hidden_profile_likes_enabled": false, "highlights_tweets_tab_ui_enabled": false, "interactive_text_enabled": false, "longform_notetweets_consumption_enabled": true, @@ -64,15 +67,25 @@ const "responsive_web_graphql_exclude_directive_enabled": true, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": false, + "responsive_web_media_download_video_enabled": false, "responsive_web_text_conversations_enabled": false, + "responsive_web_twitter_article_tweet_consumption_enabled": false, "responsive_web_twitter_blue_verified_badge_is_enabled": true, "rweb_lists_timeline_redesign_enabled": true, "spaces_2022_h2_clipping": true, "spaces_2022_h2_spaces_communities": true, "standardized_nudges_misinfo": false, + "subscriptions_verification_info_enabled": true, + "subscriptions_verification_info_reason_enabled": true, + "subscriptions_verification_info_verified_since_enabled": true, + "super_follow_badge_privacy_enabled": false, + "super_follow_exclusive_tweet_notifications_enabled": false, + "super_follow_tweet_api_enabled": false, + "super_follow_user_api_enabled": false, "tweet_awards_web_tipping_enabled": false, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false, "tweetypie_unmention_optimization_enabled": false, + "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false, "verified_phone_label_enabled": false, "vibe_api_enabled": false, "view_counts_everywhere_api_enabled": false @@ -81,41 +94,15 @@ const tweetVariables* = """{ "focalTweetId": "$1", $2 - "withBirdwatchNotes": false, - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false -}""" - - tweetResultVariables* = """{ - "tweetId": "$1", - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false, - "withCommunity": false + "includeHasBirdwatchNotes": false }""" userTweetsVariables* = """{ - "userId": "$1", $2 - "count": 20, - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false, - "withV2Timeline": true + "rest_id": "$1", $2 + "count": 20 }""" listTweetsVariables* = """{ - "listId": "$1", $2 - "count": 20, - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false + "rest_id": "$1", $2 + "count": 20 }""" diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index 0f08c4f..b9da7c4 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -9,12 +9,12 @@ proc parseGraphUser*(json: string): User = let raw = json.fromJson(GraphUser) - if raw.data.user.result.reason.get("") == "Suspended": + if raw.data.userResult.result.unavailableReason.get("") == "Suspended": return User(suspended: true) - result = toUser raw.data.user.result.legacy - result.id = raw.data.user.result.restId - result.verified = result.verified or raw.data.user.result.isBlueVerified + result = toUser raw.data.userResult.result.legacy + result.id = raw.data.userResult.result.restId + result.verified = result.verified or raw.data.userResult.result.isBlueVerified proc parseGraphListMembers*(json, cursor: string): Result[User] = result = Result[User]( diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index 478e7f3..c30eed9 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -3,7 +3,7 @@ import user type GraphUser* = object - data*: tuple[user: UserData] + data*: tuple[userResult: UserData] UserData* = object result*: UserResult @@ -12,4 +12,4 @@ type legacy*: RawUser restId*: string isBlueVerified*: bool - reason*: Option[string] + unavailableReason*: Option[string] diff --git a/src/parser.nim b/src/parser.nim index 38dbb24..7b178f3 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -29,7 +29,7 @@ proc parseUser(js: JsonNode; id=""): User = result.expandUserEntities(js) proc parseGraphUser(js: JsonNode): User = - let user = ? js{"user_results", "result"} + let user = ? js{"user_result", "result"} result = parseUser(user{"legacy"}) if "is_blue_verified" in user: @@ -262,6 +262,11 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.gif = some(parseGif(m)) else: discard + with url, m{"url"}: + if result.text.endsWith(url.getStr): + result.text.removeSuffix(url.getStr) + result.text = result.text.strip() + with jsWithheld, js{"withheld_in_countries"}: let withheldInCountries: seq[string] = if jsWithheld.kind != JArray: @[] @@ -294,16 +299,6 @@ proc finalizeTweet(global: GlobalObjects; id: string): Tweet = else: result.retweet = some Tweet() -proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = - let pin = js{"pinEntry", "entry", "entryId"}.getStr - if pin.len == 0: return - - let id = pin.getId - if id notin global.tweets: return - - global.tweets[id].pinned = true - return finalizeTweet(global, id) - proc parseGlobalObjects(js: JsonNode): GlobalObjects = result = GlobalObjects() let @@ -314,7 +309,7 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects = result.users[k] = parseUser(v, k) for k, v in tweets: - var tweet = parseTweet(v, v{"card"}) + var tweet = parseTweet(v, v{"tweet_card"}) if tweet.user.id in result.users: tweet.user = result.users[tweet.user.id] result.tweets[k] = tweet @@ -324,11 +319,6 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod return for i in js: - when T is Tweet: - if res.beginning and i{"pinEntry"}.notNull: - with pin, parsePin(i, global): - res.content.add pin - with r, i{"replaceEntry", "entry"}: if "top" in r{"entryId"}.getStr: res.top = r.getCursor @@ -369,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = proc parsePhotoRail*(js: JsonNode): PhotoRail = for tweet in js: let - t = parseTweet(tweet, js{"card"}) + t = parseTweet(tweet, js{"tweet_card"}) url = if t.photos.len > 0: t.photos[0] elif t.video.isSome: get(t.video).thumb elif t.gif.isSome: get(t.gif).thumb @@ -387,13 +377,17 @@ proc parseGraphTweet(js: JsonNode): Tweet = of "TweetUnavailable": return Tweet() of "TweetTombstone": - return Tweet(text: js{"tombstone", "text"}.getTombstone) + with text, js{"tombstone", "richText"}: + return Tweet(text: text.getTombstone) + with text, js{"tombstone", "text"}: + return Tweet(text: text.getTombstone) + return Tweet() of "TweetPreviewDisplay": return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.") of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}) - var jsCard = copy(js{"card", "legacy"}) + var jsCard = copy(js{"tweet_card", "legacy"}) if jsCard.kind != JNull: var values = newJObject() for val in jsCard["binding_values"]: @@ -401,6 +395,7 @@ proc parseGraphTweet(js: JsonNode): Tweet = jsCard["binding_values"] = values result = parseTweet(js{"legacy"}, jsCard) + result.id = js{"rest_id"}.getId result.user = parseGraphUser(js{"core"}) with noteTweet, js{"note_tweet", "note_tweet_results", "result"}: @@ -414,32 +409,31 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = for t in js{"content", "items"}: let entryId = t{"entryId"}.getStr if "cursor-showmore" in entryId: - let cursor = t{"item", "itemContent", "value"} + let cursor = t{"item", "content", "value"} result.thread.cursor = cursor.getStr result.thread.hasMore = true elif "tweet" in entryId: - let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"}) + let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"}) result.thread.content.add tweet - if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread": + if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread": result.self = true proc parseGraphTweetResult*(js: JsonNode): Tweet = - with tweet, js{"data", "tweetResult", "result"}: + with tweet, js{"data", "tweet_result", "result"}: result = parseGraphTweet(tweet) proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) - let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"} + let instructions = ? js{"data", "timeline_response", "instructions"} if instructions.len == 0: return for e in instructions[0]{"entries"}: let entryId = e{"entryId"}.getStr - # echo entryId if entryId.startsWith("tweet"): - with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: + with tweetResult, e{"content", "content", "tweetResult", "result"}: let tweet = parseGraphTweet(tweetResult) if not tweet.available: @@ -454,7 +448,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = let tweet = Tweet( id: parseBiggestInt(id), available: false, - text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone + text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone ) if id == tweetId: @@ -468,34 +462,42 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = else: result.replies.content.add thread elif entryId.startsWith("cursor-bottom"): - result.replies.bottom = e{"content", "itemContent", "value"}.getStr + result.replies.bottom = e{"content", "content", "value"}.getStr -proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline = - result = Timeline(beginning: after.len == 0) +proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = + result = Profile(tweets: Timeline(beginning: after.len == 0)) let instructions = - if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"} - else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"} + if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"} + else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} if instructions.len == 0: return for i in instructions: - if i{"type"}.getStr == "TimelineAddEntries": + if i{"__typename"}.getStr == "TimelineAddEntries": for e in i{"entries"}: let entryId = e{"entryId"}.getStr if entryId.startsWith("tweet"): - with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: + with tweetResult, e{"content", "content", "tweetResult", "result"}: let tweet = parseGraphTweet(tweetResult) if not tweet.available: tweet.id = parseBiggestInt(entryId.getId()) - result.content.add tweet - elif entryId.startsWith("profile-conversation") or entryId.startsWith("homeConversation"): + result.tweets.content.add tweet + elif "-conversation-" in entryId or entryId.startsWith("homeConversation"): let (thread, self) = parseGraphThread(e) - for tweet in thread.content: - result.content.add tweet + result.tweets.content.add thread elif entryId.startsWith("cursor-bottom"): - result.bottom = e{"content", "value"}.getStr + result.tweets.bottom = e{"content", "value"}.getStr + if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry": + with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}: + let tweet = parseGraphTweet(tweetResult) + tweet.pinned = true + if not tweet.available and tweet.tombstone.len == 0: + let entryId = i{"entry", "entryId"}.getEntryId + if entryId.len > 0: + tweet.id = parseBiggestInt(entryId) + result.pinned = some tweet proc parseGraphSearch*(js: JsonNode; after=""): Timeline = result = Timeline(beginning: after.len == 0) diff --git a/src/routes/embed.nim b/src/routes/embed.nim index baaec68..994364b 100644 --- a/src/routes/embed.nim +++ b/src/routes/embed.nim @@ -10,22 +10,22 @@ export api, embed, vdom, tweet, general, router_utils proc createEmbedRouter*(cfg: Config) = router embed: get "/i/videos/tweet/@id": - let convo = await getTweet(@"id") - if convo == nil or convo.tweet == nil or convo.tweet.video.isNone: + let tweet = await getGraphTweetResult(@"id") + if tweet == nil or tweet.video.isNone: resp Http404 - resp renderVideoEmbed(convo.tweet, cfg, request) + resp renderVideoEmbed(tweet, cfg, request) get "/@user/status/@id/embed": let - convo = await getTweet(@"id") + tweet = await getGraphTweetResult(@"id") prefs = cookiePrefs() path = getPath() - if convo == nil or convo.tweet == nil: + if tweet == nil: resp Http404 - resp renderTweetEmbed(convo.tweet, path, prefs, cfg, request) + resp renderTweetEmbed(tweet, path, prefs, cfg, request) get "/embed/Tweet.html": let id = @"id" diff --git a/src/routes/rss.nim b/src/routes/rss.nim index 1323ed3..8eec399 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -27,14 +27,12 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. else: var q = query q.fromUser = names - profile = Profile( - tweets: await getGraphSearch(q, after), - # this is kinda dumb - user: User( - username: name, - fullname: names.join(" | "), - userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png" - ) + profile = await getGraphSearch(q, after) + # this is kinda dumb + profile.user = User( + username: name, + fullname: names.join(" | "), + userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png" ) if profile.user.suspended: @@ -61,29 +59,29 @@ template respRss*(rss, page) = proc createRssRouter*(cfg: Config) = router rss: - get "/search/rss": - cond cfg.enableRss - if @"q".len > 200: - resp Http400, showError("Search input too long.", cfg) + # get "/search/rss": + # cond cfg.enableRss + # if @"q".len > 200: + # resp Http400, showError("Search input too long.", cfg) - let query = initQuery(params(request)) - if query.kind != tweets: - resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) + # let query = initQuery(params(request)) + # if query.kind != tweets: + # resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) - let - cursor = getCursor() - key = redisKey("search", $hash(genQueryUrl(query)), cursor) + # let + # cursor = getCursor() + # key = redisKey("search", $hash(genQueryUrl(query)), cursor) - var rss = await getCachedRss(key) - if rss.cursor.len > 0: - respRss(rss, "Search") + # var rss = await getCachedRss(key) + # if rss.cursor.len > 0: + # respRss(rss, "Search") - let tweets = await getGraphSearch(query, cursor) - rss.cursor = tweets.bottom - rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) + # let tweets = await getGraphSearch(query, cursor) + # rss.cursor = tweets.bottom + # rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) - await cacheRss(key, rss) - respRss(rss, "Search") + # await cacheRss(key, rss) + # respRss(rss, "Search") get "/@name/rss": cond cfg.enableRss @@ -112,7 +110,7 @@ proc createRssRouter*(cfg: Config) = case tab of "with_replies": getReplyQuery(name) of "media": getMediaQuery(name) - of "search": initQuery(params(request), name=name) + # of "search": initQuery(params(request), name=name) else: Query(fromUser: @[name]) let searchKey = if tab != "search": "" diff --git a/src/routes/search.nim b/src/routes/search.nim index 02c14e3..ed2c397 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -34,11 +34,15 @@ proc createSearchRouter*(cfg: Config) = users = Result[User](beginning: true, query: query) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title) of tweets: - let - tweets = await getGraphSearch(query, getCursor()) - rss = "/search/rss?" & genQueryUrl(query) - resp renderMain(renderTweetSearch(tweets, prefs, getPath()), - request, cfg, prefs, title, rss=rss) + # let + # tweets = await getGraphSearch(query, getCursor()) + # rss = "/search/rss?" & genQueryUrl(query) + # resp renderMain(renderTweetSearch(tweets, prefs, getPath()), + # request, cfg, prefs, title, rss=rss) + var fakeTimeline = Timeline(beginning: true) + fakeTimeline.content.add Tweet(tombstone: "Tweet search is unavailable for now") + + resp renderMain(renderTweetSearch(fakeTimeline, prefs, getPath()), request, cfg, prefs, title) else: resp Http404, showError("Invalid search", cfg) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 331b8ae..4ac60d2 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -45,34 +45,24 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; after.setLen 0 let - timeline = - case query.kind - of posts: getGraphUserTweets(userId, TimelineKind.tweets, after) - of replies: getGraphUserTweets(userId, TimelineKind.replies, after) - of media: getGraphUserTweets(userId, TimelineKind.media, after) - else: getGraphSearch(query, after) - rail = skipIf(skipRail or query.kind == media, @[]): getCachedPhotoRail(name) - user = await getCachedUser(name) + user = getCachedUser(name) - var pinned: Option[Tweet] - if not skipPinned and user.pinnedTweet > 0 and - after.len == 0 and query.kind in {posts, replies}: - let tweet = await getCachedTweet(user.pinnedTweet) - if not tweet.isNil: - tweet.pinned = true - tweet.user = user - pinned = some tweet + result = + case query.kind + of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) + of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) + of media: await getGraphUserTweets(userId, TimelineKind.media, after) + else: Profile(tweets: Timeline(beginning: true, content: @[Chain(content: + @[Tweet(tombstone: "Tweet search is unavailable for now")] + )])) + # else: await getGraphSearch(query, after) - result = Profile( - user: user, - pinned: pinned, - tweets: await timeline, - photoRail: await rail - ) + result.user = await user + result.photoRail = await rail if result.user.protected or result.user.suspended: return @@ -83,8 +73,11 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = if query.fromUser.len != 1: let - timeline = await getGraphSearch(query, after) - html = renderTweetSearch(timeline, prefs, getPath()) + # timeline = await getGraphSearch(query, after) + timeline = Profile(tweets: Timeline(beginning: true, content: @[Chain(content: + @[Tweet(tombstone: "This features is unavailable for now")] + )])) + html = renderTweetSearch(timeline.tweets, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins) @@ -138,7 +131,7 @@ proc createTimelineRouter*(cfg: Config) = # used for the infinite scroll feature if @"scroll".len > 0: if query.fromUser.len != 1: - var timeline = await getGraphSearch(query, after) + var timeline = (await getGraphSearch(query, after)).tweets if timeline.content.len == 0: resp Http404 timeline.beginning = true resp $renderTweetSearch(timeline, prefs, getPath()) diff --git a/src/sass/tweet/thread.scss b/src/sass/tweet/thread.scss index 5fbad21..19fb3e0 100644 --- a/src/sass/tweet/thread.scss +++ b/src/sass/tweet/thread.scss @@ -110,3 +110,29 @@ margin-left: 58px; padding: 7px 0; } + +.timeline-item.thread.more-replies-thread { + padding: 0 0.75em; + + &::before { + top: 40px; + margin-bottom: 31px; + } + + .more-replies { + display: flex; + padding-top: unset !important; + margin-top: 8px; + + &::before { + display: inline-block; + position: relative; + top: -1px; + line-height: 0.4em; + } + + .more-replies-text { + display: inline; + } + } +} diff --git a/src/tokens.nim b/src/tokens.nim index 6ef81f5..6643de3 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,11 +41,10 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.timeline: 187 - of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets, - Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, - Api.userRestId, Api.userScreenName, - Api.tweetDetail, Api.tweetResult, Api.search: 500 + of Api.timeline: 180 + of Api.userTweets, Api.userTweetsAndReplies, Api.userRestId, + Api.userScreenName, Api.tweetDetail, Api.tweetResult, Api.search: 500 + of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 of Api.userSearch: 900 reqs = maxReqs - token.apis[api].remaining diff --git a/src/types.nim b/src/types.nim index 4dca5f0..e7f3303 100644 --- a/src/types.nim +++ b/src/types.nim @@ -222,7 +222,7 @@ type after*: Chain replies*: Result[Chain] - Timeline* = Result[Tweet] + Timeline* = Result[Chain] Profile* = object user*: User @@ -274,3 +274,6 @@ type proc contains*(thread: Chain; tweet: Tweet): bool = thread.content.anyIt(it.id == tweet.id) + +proc add*(timeline: var seq[Chain]; tweet: Tweet) = + timeline.add Chain(content: @[tweet]) diff --git a/src/views/rss.nimf b/src/views/rss.nimf index 96f6466..ce2518a 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -56,24 +56,29 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname} #end if #end proc # -#proc renderRssTweets(tweets: seq[Tweet]; cfg: Config): string = +#proc renderRssTweets(tweets: seq[Chain]; cfg: Config; userId=""): string = #let urlPrefix = getUrlPrefix(cfg) #var links: seq[string] -#for t in tweets: -# let retweet = if t.retweet.isSome: t.user.username else: "" -# let tweet = if retweet.len > 0: t.retweet.get else: t -# let link = getLink(tweet) -# if link in links: continue -# end if -# links.add link - - ${getTitle(tweet, retweet)} - @${tweet.user.username} - - ${getRfc822Time(tweet)} - ${urlPrefix & link} - ${urlPrefix & link} - +#for c in tweets: +# for t in c.content: +# if userId.len > 0 and t.user.id != userId: continue +# end if +# +# let retweet = if t.retweet.isSome: t.user.username else: "" +# let tweet = if retweet.len > 0: t.retweet.get else: t +# let link = getLink(tweet) +# if link in links: continue +# end if +# links.add link + + ${getTitle(tweet, retweet)} + @${tweet.user.username} + + ${getRfc822Time(tweet)} + ${urlPrefix & link} + ${urlPrefix & link} + +# end for #end for #end proc # @@ -102,13 +107,13 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname} 128 #if profile.tweets.content.len > 0: -${renderRssTweets(profile.tweets.content, cfg)} +${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)} #end if #end proc # -#proc renderListRss*(tweets: seq[Tweet]; list: List; cfg: Config): string = +#proc renderListRss*(tweets: seq[Chain]; list: List; cfg: Config): string = #let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}" #result = "" @@ -125,7 +130,7 @@ ${renderRssTweets(tweets, cfg)} #end proc # -#proc renderSearchRss*(tweets: seq[Tweet]; name, param: string; cfg: Config): string = +#proc renderSearchRss*(tweets: seq[Chain]; name, param: string; cfg: Config): string = #let link = &"{getUrlPrefix(cfg)}/search" #let escName = xmltree.escape(name) #result = "" diff --git a/src/views/search.nim b/src/views/search.nim index 72c59f5..401e6da 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -88,7 +88,7 @@ proc renderSearchPanel*(query: Query): VNode = span(class="search-title"): text "Near" genInput("near", "", query.near, "Location...", autofocus=false) -proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string; +proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string; pinned=none(Tweet)): VNode = let query = results.query buildHtml(tdiv(class="timeline-container")): diff --git a/src/views/timeline.nim b/src/views/timeline.nim index 54cad7a..8ae888e 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, strformat, sequtils, algorithm, uri, options +import strutils, strformat, algorithm, uri, options import karax/[karaxdsl, vdom] import ".."/[types, query, formatters] @@ -43,20 +43,18 @@ proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode = buildHtml(tdiv(class="thread-line")): let sortedThread = thread.sortedByIt(it.id) for i, tweet in sortedThread: + # thread has a gap, display "more replies" link + if i > 0 and tweet.replyId != sortedThread[i - 1].id: + tdiv(class="timeline-item thread more-replies-thread"): + tdiv(class="more-replies"): + a(class="more-replies-text", href=getLink(tweet)): + text "more replies" + let show = i == thread.high and sortedThread[0].id != tweet.threadId let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: "" renderTweet(tweet, prefs, path, class=(header & "thread"), index=i, last=(i == thread.high), showThread=show) -proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] = - result = @[it] - if it.retweet.isSome or it.replyId in threads: return - for t in tweets: - if t.id == result[0].replyId: - result.insert t - elif t.replyId == result[0].id: - result.add t - proc renderUser(user: User; prefs: Prefs): VNode = buildHtml(tdiv(class="timeline-item")): a(class="tweet-link", href=("/" & user.username)) @@ -89,7 +87,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode = else: renderNoMore() -proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string; +proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string; pinned=none(Tweet)): VNode = buildHtml(tdiv(class="timeline")): if not results.beginning: @@ -105,26 +103,26 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string; else: renderNoneFound() else: - var - threads: seq[int64] - retweets: seq[int64] + var retweets: seq[int64] - for tweet in results.content: - let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0 + for thread in results.content: + if thread.content.len == 1: + let + tweet = thread.content[0] + retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0 - if tweet.id in threads or rt in retweets or tweet.id in retweets or - tweet.pinned and prefs.hidePins: continue + if retweetId in retweets or tweet.id in retweets or + tweet.pinned and prefs.hidePins: + continue - let thread = results.content.threadFilter(threads, tweet) - if thread.len < 2: var hasThread = tweet.hasThread - if rt != 0: - retweets &= rt + if retweetId != 0 and tweet.retweet.isSome: + retweets &= retweetId hasThread = get(tweet.retweet).hasThread renderTweet(tweet, prefs, path, showThread=hasThread) else: - renderThread(thread, prefs, path) - threads &= thread.mapIt(it.id) + renderThread(thread.content, prefs, path) - renderMore(results.query, results.bottom) + if results.bottom.len > 0: + renderMore(results.query, results.bottom) renderToTop() diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 3338b71..f47ae9a 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -14,15 +14,14 @@ proc renderMiniAvatar(user: User; prefs: Prefs): VNode = buildHtml(): img(class=(prefs.getAvatarClass & " mini"), src=url) -proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode = +proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode = buildHtml(tdiv): - if retweet.len > 0: - tdiv(class="retweet-header"): - span: icon "retweet", retweet & " retweeted" - - if tweet.pinned: + if pinned: tdiv(class="pinned"): span: icon "pin", "Pinned Tweet" + elif retweet.len > 0: + tdiv(class="retweet-header"): + span: icon "retweet", retweet & " retweeted" tdiv(class="tweet-header"): a(class="tweet-avatar", href=("/" & tweet.user.username)): @@ -290,7 +289,10 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; if tweet.quote.isSome: renderQuote(tweet.quote.get(), prefs, path) - let fullTweet = tweet + let + fullTweet = tweet + pinned = tweet.pinned + var retweet: string var tweet = fullTweet if tweet.retweet.isSome: @@ -303,7 +305,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; tdiv(class="tweet-body"): var views = "" - renderHeader(tweet, retweet, prefs) + renderHeader(tweet, retweet, pinned, prefs) if not afterTweet and index == 0 and tweet.reply.len > 0 and (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username): diff --git a/tests/test_card.py b/tests/test_card.py index 696b9d5..f84ddca 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -16,7 +16,12 @@ card = [ ['FluentAI/status/1116417904831029248', 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', 'One of the only ways to improve Alexa is to have human beings check it for errors', - 'theverge.com', True] + 'theverge.com', True], + + ['nim_lang/status/1082989146040340480', + 'Nim in 2018: A short recap', + 'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.', + 'nim-lang.org', True] ] no_thumb = [ @@ -33,12 +38,7 @@ no_thumb = [ ['voidtarget/status/1133028231672582145', 'sinkingsugar/nimqt-example', 'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.', - 'github.com'], - - ['nim_lang/status/1082989146040340480', - 'Nim in 2018: A short recap', - 'Posted by u/miran1 - 36 votes and 46 comments', - 'reddit.com'] + 'github.com'] ] playable = [ @@ -53,17 +53,6 @@ playable = [ 'youtube.com'] ] -# promo = [ - # ['BangOlufsen/status/1145698701517754368', - # 'Upgrade your journey', '', - # 'www.bang-olufsen.com'], - - # ['BangOlufsen/status/1154934429900406784', - # 'Learn more about Beosound Shape', '', - # 'www.bang-olufsen.com'] -# ] - - class CardTest(BaseTestCase): @parameterized.expand(card) def test_card(self, tweet, title, description, destination, large): @@ -98,13 +87,3 @@ class CardTest(BaseTestCase): self.assert_element_visible('.card-overlay') if len(description) > 0: self.assert_text(description, c.description) - - # @parameterized.expand(promo) - # def test_card_promo(self, tweet, title, description, destination): - # self.open_nitter(tweet) - # c = Card(Conversation.main + " ") - # self.assert_text(title, c.title) - # self.assert_text(destination, c.destination) - # self.assert_element_visible('.video-overlay') - # if len(description) > 0: - # self.assert_text(description, c.description) diff --git a/tests/test_profile.py b/tests/test_profile.py index f9b5047..4c75ad2 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -66,8 +66,8 @@ class ProfileTest(BaseTestCase): self.assert_text(f'User "{username}" not found') def test_suspended(self): - self.open_nitter('user') - self.assert_text('User "user" has been suspended') + self.open_nitter('suspendme') + self.assert_text('User "suspendme" has been suspended') @parameterized.expand(banner_image) def test_banner_image(self, username, url): diff --git a/tests/test_search.py b/tests/test_search.py index 80ee36a..62c4640 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -2,8 +2,8 @@ from base import BaseTestCase from parameterized import parameterized -class SearchTest(BaseTestCase): - @parameterized.expand([['@mobile_test'], ['@mobile_test_2']]) - def test_username_search(self, username): - self.search_username(username) - self.assert_text(f'{username}') +#class SearchTest(BaseTestCase): + #@parameterized.expand([['@mobile_test'], ['@mobile_test_2']]) + #def test_username_search(self, username): + #self.search_username(username) + #self.assert_text(f'{username}') diff --git a/tests/test_tweet.py b/tests/test_tweet.py index 9209e70..e4231a4 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -74,9 +74,9 @@ retweet = [ [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr'] ] -reply = [ - ['mobile_test/with_replies', 15] -] +# reply = [ +# ['mobile_test/with_replies', 15] +# ] class TweetTest(BaseTestCase): @@ -137,8 +137,8 @@ class TweetTest(BaseTestCase): self.open_nitter(tweet) self.assert_text('Tweet not found', '.error-panel') - @parameterized.expand(reply) - def test_thread(self, tweet, num): - self.open_nitter(tweet) - thread = self.find_element(f'.timeline > div:nth-child({num})') - self.assertIn(thread.get_attribute('class'), 'thread-line') + # @parameterized.expand(reply) + # def test_thread(self, tweet, num): + # self.open_nitter(tweet) + # thread = self.find_element(f'.timeline > div:nth-child({num})') + # self.assertIn(thread.get_attribute('class'), 'thread-line') From b290f6fd29ac20717bad359fc55d32822d3e054d Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 12 Jul 2023 01:34:39 +0200 Subject: [PATCH 12/14] Optimize timeline data structure --- src/parser.nim | 2 +- src/routes/timeline.nim | 8 ++------ src/types.nim | 10 ++++++---- src/views/rss.nimf | 16 ++++++++-------- src/views/timeline.nim | 8 ++++---- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/parser.nim b/src/parser.nim index 7b178f3..5dc96df 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -486,7 +486,7 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = result.tweets.content.add tweet elif "-conversation-" in entryId or entryId.startsWith("homeConversation"): let (thread, self) = parseGraphThread(e) - result.tweets.content.add thread + result.tweets.content.add thread.content elif entryId.startsWith("cursor-bottom"): result.tweets.bottom = e{"content", "value"}.getStr if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry": diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 4ac60d2..e62c9e0 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -56,9 +56,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) - else: Profile(tweets: Timeline(beginning: true, content: @[Chain(content: - @[Tweet(tombstone: "Tweet search is unavailable for now")] - )])) + else: Profile(tweets: Timeline(beginning: true, content: @[@[Tweet(tombstone: "Tweet search is unavailable for now")]])) # else: await getGraphSearch(query, after) result.user = await user @@ -74,9 +72,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; if query.fromUser.len != 1: let # timeline = await getGraphSearch(query, after) - timeline = Profile(tweets: Timeline(beginning: true, content: @[Chain(content: - @[Tweet(tombstone: "This features is unavailable for now")] - )])) + timeline = Profile(tweets: Timeline(beginning: true, content: @[@[Tweet(tombstone: "This features is unavailable for now")]])) html = renderTweetSearch(timeline.tweets, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) diff --git a/src/types.nim b/src/types.nim index e7f3303..f7d5f6b 100644 --- a/src/types.nim +++ b/src/types.nim @@ -205,6 +205,8 @@ type video*: Option[Video] photos*: seq[string] + Tweets* = seq[Tweet] + Result*[T] = object content*: seq[T] top*, bottom*: string @@ -212,7 +214,7 @@ type query*: Query Chain* = object - content*: seq[Tweet] + content*: Tweets hasMore*: bool cursor*: string @@ -222,7 +224,7 @@ type after*: Chain replies*: Result[Chain] - Timeline* = Result[Chain] + Timeline* = Result[Tweets] Profile* = object user*: User @@ -275,5 +277,5 @@ type proc contains*(thread: Chain; tweet: Tweet): bool = thread.content.anyIt(it.id == tweet.id) -proc add*(timeline: var seq[Chain]; tweet: Tweet) = - timeline.add Chain(content: @[tweet]) +proc add*(timeline: var seq[Tweets]; tweet: Tweet) = + timeline.add @[tweet] diff --git a/src/views/rss.nimf b/src/views/rss.nimf index ce2518a..036a7b9 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -56,16 +56,16 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname} #end if #end proc # -#proc renderRssTweets(tweets: seq[Chain]; cfg: Config; userId=""): string = +#proc renderRssTweets(tweets: seq[Tweets]; cfg: Config; userId=""): string = #let urlPrefix = getUrlPrefix(cfg) #var links: seq[string] -#for c in tweets: -# for t in c.content: -# if userId.len > 0 and t.user.id != userId: continue +#for thread in tweets: +# for tweet in thread: +# if userId.len > 0 and tweet.user.id != userId: continue # end if # -# let retweet = if t.retweet.isSome: t.user.username else: "" -# let tweet = if retweet.len > 0: t.retweet.get else: t +# let retweet = if tweet.retweet.isSome: tweet.user.username else: "" +# let tweet = if retweet.len > 0: tweet.retweet.get else: tweet # let link = getLink(tweet) # if link in links: continue # end if @@ -113,7 +113,7 @@ ${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)} #end proc # -#proc renderListRss*(tweets: seq[Chain]; list: List; cfg: Config): string = +#proc renderListRss*(tweets: seq[Tweets]; list: List; cfg: Config): string = #let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}" #result = "" @@ -130,7 +130,7 @@ ${renderRssTweets(tweets, cfg)} #end proc # -#proc renderSearchRss*(tweets: seq[Chain]; name, param: string; cfg: Config): string = +#proc renderSearchRss*(tweets: seq[Tweets]; name, param: string; cfg: Config): string = #let link = &"{getUrlPrefix(cfg)}/search" #let escName = xmltree.escape(name) #result = "" diff --git a/src/views/timeline.nim b/src/views/timeline.nim index 8ae888e..abeb6d3 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -39,7 +39,7 @@ proc renderNoneFound(): VNode = h2(class="timeline-none"): text "No items found" -proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode = +proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode = buildHtml(tdiv(class="thread-line")): let sortedThread = thread.sortedByIt(it.id) for i, tweet in sortedThread: @@ -106,9 +106,9 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string; var retweets: seq[int64] for thread in results.content: - if thread.content.len == 1: + if thread.len == 1: let - tweet = thread.content[0] + tweet = thread[0] retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0 if retweetId in retweets or tweet.id in retweets or @@ -121,7 +121,7 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string; hasThread = get(tweet.retweet).hasThread renderTweet(tweet, prefs, path, showThread=hasThread) else: - renderThread(thread.content, prefs, path) + renderThread(thread, prefs, path) if results.bottom.len > 0: renderMore(results.query, results.bottom) From 67203a431d9242e2f0d5fbdbfa86ba55f0a4cc54 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 12 Jul 2023 03:37:44 +0200 Subject: [PATCH 13/14] Add back search --- src/api.nim | 16 ++++++++++++++++ src/consts.nim | 1 + src/parser.nim | 34 ++++++++++++++++++++++++++++++++-- src/routes/rss.nim | 38 +++++++++++++++++++------------------- src/routes/search.nim | 14 +++++--------- src/routes/timeline.nim | 8 +++----- src/tokens.nim | 4 ++-- 7 files changed, 78 insertions(+), 37 deletions(-) diff --git a/src/api.nim b/src/api.nim index d99eb3d..60af68d 100644 --- a/src/api.nim +++ b/src/api.nim @@ -115,6 +115,22 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} = result = Profile(tweets: parseGraphSearch(await fetch(url, Api.search), after)) result.tweets.query = query +proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = + let q = genQueryParam(query) + if q.len == 0 or q == emptyQuery: + return Timeline(query: query, beginning: true) + + let url = tweetSearch ? genParams({ + "q": q, + "tweet_search_mode": "live", + "max_id": after + }) + + result = parseTweetSearch(await fetch(url, Api.search)) + result.query = query + if after.len == 0: + result.beginning = true + proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = if query.text.len == 0: return Result[User](query: query, beginning: true) diff --git a/src/consts.nim b/src/consts.nim index 184f9da..8dd1b14 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -9,6 +9,7 @@ const photoRail* = api / "1.1/statuses/media_timeline.json" userSearch* = api / "1.1/users/search.json" + tweetSearch* = api / "1.1/search/tweets.json" graphql = api / "graphql" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" diff --git a/src/parser.nim b/src/parser.nim index 5dc96df..f298160 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -82,12 +82,16 @@ proc parseVideo(js: JsonNode): Video = result = Video( thumb: js{"media_url_https"}.getImageStr, views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt), - available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available", + available: true, title: js{"ext_alt_text"}.getStr, durationMs: js{"video_info", "duration_millis"}.getInt # playbackType: mp4 ) + with status, js{"ext_media_availability", "status"}: + if status.getStr.len > 0 and status.getStr.toLowerAscii != "available": + result.available = false + with title, js{"additional_media_info", "title"}: result.title = title.getStr @@ -219,7 +223,9 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = if result.hasThread and result.threadId == 0: result.threadId = js{"self_thread", "id_str"}.getId - if js{"is_quote_status"}.getBool: + if "retweeted_status" in js: + result.retweet = some Tweet() + elif js{"is_quote_status"}.getBool: result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId) # legacy @@ -281,6 +287,30 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.text.removeSuffix(" Learn more.") result.available = false +proc parseLegacyTweet(js: JsonNode): Tweet = + result = parseTweet(js, js{"card"}) + if not result.isNil and result.available: + result.user = parseUser(js{"user"}) + + if result.quote.isSome: + result.quote = some parseLegacyTweet(js{"quoted_status"}) + +proc parseTweetSearch*(js: JsonNode): Timeline = + if js.kind == JNull or "statuses" notin js: + return Timeline(beginning: true) + + for tweet in js{"statuses"}: + let parsed = parseLegacyTweet(tweet) + + if parsed.retweet.isSome: + parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"}) + + result.content.add @[parsed] + + let cursor = js{"search_metadata", "next_results"}.getStr + if cursor.len > 0 and "max_id" in cursor: + result.bottom = cursor[cursor.find("=") + 1 .. cursor.find("&q=")] + proc finalizeTweet(global: GlobalObjects; id: string): Tweet = let intId = if id.len > 0: parseBiggestInt(id) else: 0 result = global.tweets.getOrDefault(id, Tweet(id: intId)) diff --git a/src/routes/rss.nim b/src/routes/rss.nim index 8eec399..d378396 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -27,7 +27,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. else: var q = query q.fromUser = names - profile = await getGraphSearch(q, after) + profile.tweets = await getTweetSearch(q, after) # this is kinda dumb profile.user = User( username: name, @@ -59,29 +59,29 @@ template respRss*(rss, page) = proc createRssRouter*(cfg: Config) = router rss: - # get "/search/rss": - # cond cfg.enableRss - # if @"q".len > 200: - # resp Http400, showError("Search input too long.", cfg) + get "/search/rss": + cond cfg.enableRss + if @"q".len > 200: + resp Http400, showError("Search input too long.", cfg) - # let query = initQuery(params(request)) - # if query.kind != tweets: - # resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) + let query = initQuery(params(request)) + if query.kind != tweets: + resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) - # let - # cursor = getCursor() - # key = redisKey("search", $hash(genQueryUrl(query)), cursor) + let + cursor = getCursor() + key = redisKey("search", $hash(genQueryUrl(query)), cursor) - # var rss = await getCachedRss(key) - # if rss.cursor.len > 0: - # respRss(rss, "Search") + var rss = await getCachedRss(key) + if rss.cursor.len > 0: + respRss(rss, "Search") - # let tweets = await getGraphSearch(query, cursor) - # rss.cursor = tweets.bottom - # rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) + let tweets = await getTweetSearch(query, cursor) + rss.cursor = tweets.bottom + rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) - # await cacheRss(key, rss) - # respRss(rss, "Search") + await cacheRss(key, rss) + respRss(rss, "Search") get "/@name/rss": cond cfg.enableRss diff --git a/src/routes/search.nim b/src/routes/search.nim index ed2c397..c270df5 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -34,15 +34,11 @@ proc createSearchRouter*(cfg: Config) = users = Result[User](beginning: true, query: query) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title) of tweets: - # let - # tweets = await getGraphSearch(query, getCursor()) - # rss = "/search/rss?" & genQueryUrl(query) - # resp renderMain(renderTweetSearch(tweets, prefs, getPath()), - # request, cfg, prefs, title, rss=rss) - var fakeTimeline = Timeline(beginning: true) - fakeTimeline.content.add Tweet(tombstone: "Tweet search is unavailable for now") - - resp renderMain(renderTweetSearch(fakeTimeline, prefs, getPath()), request, cfg, prefs, title) + let + tweets = await getTweetSearch(query, getCursor()) + rss = "/search/rss?" & genQueryUrl(query) + resp renderMain(renderTweetSearch(tweets, prefs, getPath()), + request, cfg, prefs, title, rss=rss) else: resp Http404, showError("Invalid search", cfg) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index e62c9e0..ef3d012 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -56,8 +56,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) - else: Profile(tweets: Timeline(beginning: true, content: @[@[Tweet(tombstone: "Tweet search is unavailable for now")]])) - # else: await getGraphSearch(query, after) + else: Profile(tweets: await getTweetSearch(query, after)) result.user = await user result.photoRail = await rail @@ -71,9 +70,8 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = if query.fromUser.len != 1: let - # timeline = await getGraphSearch(query, after) - timeline = Profile(tweets: Timeline(beginning: true, content: @[@[Tweet(tombstone: "This features is unavailable for now")]])) - html = renderTweetSearch(timeline.tweets, prefs, getPath()) + timeline = await getTweetSearch(query, after) + html = renderTweetSearch(timeline, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins) diff --git a/src/tokens.nim b/src/tokens.nim index 6643de3..531f557 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,9 +41,9 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.timeline: 180 + of Api.timeline, Api.search: 180 of Api.userTweets, Api.userTweetsAndReplies, Api.userRestId, - Api.userScreenName, Api.tweetDetail, Api.tweetResult, Api.search: 500 + Api.userScreenName, Api.tweetDetail, Api.tweetResult: 500 of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 of Api.userSearch: 900 reqs = maxReqs - token.apis[api].remaining From afbdbd293e30f614ee288731717868c6d618b55f Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 12 Jul 2023 03:47:37 +0200 Subject: [PATCH 14/14] Fix protected user photo rail crash --- src/parser.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/parser.nim b/src/parser.nim index f298160..b988cf7 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -387,6 +387,10 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = result.top = cursor{"value"}.getStr proc parsePhotoRail*(js: JsonNode): PhotoRail = + with error, js{"error"}: + if error.getStr == "Not authorized.": + return + for tweet in js: let t = parseTweet(tweet, js{"tweet_card"})