From 59a72831c749b2198cb83d1b7cee74a5d05da723 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 24 Jul 2023 04:26:32 +0200 Subject: [PATCH 01/12] Apply cached profile verified status to tweets --- src/routes/timeline.nim | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 8b8a23c..bf2a08e 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -66,13 +66,17 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; if result.user.protected or result.user.suspended: return - if not skipPinned and query.kind == posts and - result.user.pinnedTweet > 0 and after.len == 0: - let tweet = await getCachedTweet(result.user.pinnedTweet) - if not tweet.isNil: - tweet.pinned = true - tweet.user = result.user - result.pinned = some tweet + if query.kind == posts: + if result.user.verified: + for chain in result.tweets.content: + if chain[0].user.id == result.user.id: + chain[0].user.verified = true + if not skipPinned and result.user.pinnedTweet > 0 and after.len == 0: + let tweet = await getCachedTweet(result.user.pinnedTweet) + if not tweet.isNil: + tweet.pinned = true + tweet.user = result.user + result.pinned = some tweet proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = From 39192bf191dfc8c5645aa8101afd04474b899897 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 24 Jul 2023 10:18:50 +0200 Subject: [PATCH 02/12] Fix multi-timeline infinite scroll --- src/routes/timeline.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index bf2a08e..82dc45b 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -137,7 +137,7 @@ proc createTimelineRouter*(cfg: Config) = # used for the infinite scroll feature if @"scroll".len > 0: if query.fromUser.len != 1: - var timeline = (await getGraphSearch(query, after)).tweets + var timeline = await getTweetSearch(query, after) if timeline.content.len == 0: resp Http404 timeline.beginning = true resp $renderTweetSearch(timeline, prefs, getPath()) From 20b5cce5dc6437ffc06ea53e9efd884f2fc66abe Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 24 Jul 2023 10:37:25 +0200 Subject: [PATCH 03/12] Retry infinite scroll errors --- public/js/infiniteScroll.js | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/public/js/infiniteScroll.js b/public/js/infiniteScroll.js index 9939c03..be27e0c 100644 --- a/public/js/infiniteScroll.js +++ b/public/js/infiniteScroll.js @@ -5,7 +5,7 @@ function insertBeforeLast(node, elem) { } function getLoadMore(doc) { - return doc.querySelector('.show-more:not(.timeline-item)'); + return doc.querySelector(".show-more:not(.timeline-item)"); } function isDuplicate(item, itemClass) { @@ -15,18 +15,19 @@ function isDuplicate(item, itemClass) { return document.querySelector(itemClass + " .tweet-link[href='" + href + "']") != null; } -window.onload = function() { +window.onload = function () { const url = window.location.pathname; const isTweet = url.indexOf("/status/") !== -1; const containerClass = isTweet ? ".replies" : ".timeline"; - const itemClass = containerClass + ' > div:not(.top-ref)'; + const itemClass = containerClass + " > div:not(.top-ref)"; var html = document.querySelector("html"); var container = document.querySelector(containerClass); var loading = false; - window.addEventListener('scroll', function() { + function handleScroll(failed) { if (loading) return; + if (html.scrollTop + html.clientHeight >= html.scrollHeight - 3000) { loading = true; var loadMore = getLoadMore(document); @@ -35,13 +36,15 @@ window.onload = function() { loadMore.children[0].text = "Loading..."; var url = new URL(loadMore.children[0].href); - url.searchParams.append('scroll', 'true'); + url.searchParams.append("scroll", "true"); fetch(url.toString()).then(function (response) { + if (response.status === 404) throw "error"; + return response.text(); }).then(function (html) { var parser = new DOMParser(); - var doc = parser.parseFromString(html, 'text/html'); + var doc = parser.parseFromString(html, "text/html"); loadMore.remove(); for (var item of doc.querySelectorAll(itemClass)) { @@ -57,10 +60,18 @@ window.onload = function() { if (isTweet) container.appendChild(newLoadMore); else insertBeforeLast(container, newLoadMore); }).catch(function (err) { - console.warn('Something went wrong.', err); - loading = true; + console.warn("Something went wrong.", err); + if (failed > 3) { + loadMore.children[0].text = "Error"; + return; + } + + loading = false; + handleScroll((failed || 0) + 1); }); } - }); + } + + window.addEventListener("scroll", () => handleScroll()); }; // @license-end From 5725780c990cd55f00ed6558d052f0e5a148652a Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 6 Aug 2023 21:02:22 +0200 Subject: [PATCH 04/12] Bump Nim version in Docker image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c100394..138dc64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nimlang/nim:1.6.10-alpine-regular as nim +FROM nimlang/nim:2.0.0-alpine-regular as nim LABEL maintainer="setenforce@protonmail.com" RUN apk --no-cache add libsass-dev pcre From 624394430c0989d18c279153006c6a7e48f4dd03 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 8 Aug 2023 02:09:56 +0200 Subject: [PATCH 05/12] Use legacy timeline/user endpoint for Tweets tab --- src/api.nim | 10 ++++++++ src/apiutils.nim | 2 +- src/consts.nim | 23 +++++++----------- src/parser.nim | 52 +++++++++++++++++++++++++++++++++++++++-- src/parserutils.nim | 6 +++++ src/routes/timeline.nim | 17 +------------- src/tokens.nim | 8 +++---- src/types.nim | 1 + 8 files changed, 81 insertions(+), 38 deletions(-) diff --git a/src/api.nim b/src/api.nim index c7dc0e0..c313aa2 100644 --- a/src/api.nim +++ b/src/api.nim @@ -40,6 +40,16 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi # url = oldUserTweets / (id & ".json") ? ps # result = parseTimeline(await fetch(url, Api.timeline), after) +proc getUserTimeline*(id: string; after=""): Future[Profile] {.async.} = + var ps = genParams({"id": id}) + if after.len > 0: + ps.add ("down_cursor", after) + + let + url = legacyUserTweets ? ps + js = await fetch(url, Api.userTimeline) + result = parseUserTimeline(js, after) + proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let diff --git a/src/apiutils.nim b/src/apiutils.nim index c0c01d4..1da971a 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -16,8 +16,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; for p in pars: result &= p if ext: - result &= ("ext", "mediaStats,isBlueVerified,isVerified,blue,blueVerified") result &= ("include_ext_alt_text", "1") + result &= ("include_ext_media_stats", "1") result &= ("include_ext_media_availability", "1") if count.len > 0: result &= ("count", count) diff --git a/src/consts.nim b/src/consts.nim index 80a098f..a25f6ea 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -7,6 +7,7 @@ const api = parseUri("https://api.twitter.com") activate* = $(api / "1.1/guest/activate.json") + legacyUserTweets* = api / "1.1/timeline/user.json" photoRail* = api / "1.1/statuses/media_timeline.json" userSearch* = api / "1.1/users/search.json" tweetSearch* = api / "1.1/search/universal.json" @@ -28,28 +29,20 @@ const graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" timelineParams* = { - "cards_platform": "Web-13", - "tweet_mode": "extended", - "ui_lang": "en-US", - "send_error_codes": "1", - "simple_quoted_tweet": "1", - "skip_status": "1", - "include_blocked_by": "0", - "include_blocking": "0", - "include_can_dm": "0", "include_can_media_tag": "1", "include_cards": "1", - "include_composer_source": "0", "include_entities": "1", - "include_ext_is_blue_verified": "1", - "include_ext_media_color": "0", - "include_followed_by": "0", - "include_mute_edge": "0", "include_profile_interstitial_type": "0", "include_quote_count": "1", "include_reply_count": "1", "include_user_entities": "1", - "include_want_retweets": "0", + "include_ext_reply_count": "1", + "include_ext_is_blue_verified": "1", + "include_ext_media_color": "0", + "cards_platform": "Web-13", + "tweet_mode": "extended", + "send_error_codes": "1", + "simple_quoted_tweet": "1" }.toSeq gqlFeatures* = """{ diff --git a/src/parser.nim b/src/parser.nim index 991ca6e..c7d8bd1 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, options, times, math +import strutils, options, times, math, tables import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard @@ -81,7 +81,7 @@ proc parseGif(js: JsonNode): Gif = proc parseVideo(js: JsonNode): Video = result = Video( thumb: js{"media_url_https"}.getImageStr, - views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt), + views: getVideoViewCount(js), available: true, title: js{"ext_alt_text"}.getStr, durationMs: js{"video_info", "duration_millis"}.getInt @@ -313,6 +313,54 @@ proc parseTweetSearch*(js: JsonNode; after=""): Timeline = if result.content.len > 0: result.bottom = $(result.content[^1][0].id - 1) +proc parseUserTimelineTweet(tweet: JsonNode; users: TableRef[string, User]): Tweet = + result = parseTweet(tweet, tweet{"card"}) + + if result.isNil or not result.available: + return + + with user, tweet{"user"}: + let userId = user{"id_str"}.getStr + if user{"ext_is_blue_verified"}.getBool(false): + users[userId].verified = users[userId].verified or true + result.user = users[userId] + +proc parseUserTimeline*(js: JsonNode; after=""): Profile = + result = Profile(tweets: Timeline(beginning: after.len == 0)) + + if js.kind == JNull or "response" notin js or "twitter_objects" notin js: + return + + var users = newTable[string, User]() + for userId, user in js{"twitter_objects", "users"}: + users[userId] = parseUser(user) + + for entity in js{"response", "timeline"}: + let + tweetId = entity{"tweet", "id"}.getId + isPinned = entity{"tweet", "is_pinned"}.getBool(false) + + with tweet, js{"twitter_objects", "tweets", $tweetId}: + var parsed = parseUserTimelineTweet(tweet, users) + + if not parsed.isNil and parsed.available: + if parsed.quote.isSome: + parsed.quote = some parseUserTimelineTweet(tweet{"quoted_status"}, users) + + if parsed.retweet.isSome: + let retweet = parseUserTimelineTweet(tweet{"retweeted_status"}, users) + if retweet.quote.isSome: + retweet.quote = some parseUserTimelineTweet(tweet{"retweeted_status", "quoted_status"}, users) + parsed.retweet = some retweet + + if isPinned: + parsed.pinned = true + result.pinned = some parsed + else: + result.tweets.content.add parsed + + result.tweets.bottom = js{"response", "cursor", "bottom"}.getStr + # proc finalizeTweet(global: GlobalObjects; id: string): Tweet = # let intId = if id.len > 0: parseBiggestInt(id) else: 0 # result = global.tweets.getOrDefault(id, Tweet(id: intId)) diff --git a/src/parserutils.nim b/src/parserutils.nim index f28bd52..c65052e 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -148,6 +148,12 @@ proc getMp4Resolution*(url: string): int = # cannot determine resolution (e.g. m3u8/non-mp4 video) return 0 +proc getVideoViewCount*(js: JsonNode): string = + with stats, js{"ext_media_stats"}: + return stats{"view_count"}.getStr($stats{"viewCount"}.getInt) + + return $js{"mediaStats", "viewCount"}.getInt(0) + proc extractSlice(js: JsonNode): Slice[int] = result = js["indices"][0].getInt ..< js["indices"][1].getInt diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 82dc45b..8d02b68 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -53,7 +53,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result = case query.kind - # of posts: await getTimeline(userId, after) + of posts: await getUserTimeline(userId, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) else: Profile(tweets: await getTweetSearch(query, after)) @@ -63,21 +63,6 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result.tweets.query = query - if result.user.protected or result.user.suspended: - return - - if query.kind == posts: - if result.user.verified: - for chain in result.tweets.content: - if chain[0].user.id == result.user.id: - chain[0].user.verified = true - if not skipPinned and result.user.pinnedTweet > 0 and after.len == 0: - let tweet = await getCachedTweet(result.user.pinnedTweet) - if not tweet.isNil: - tweet.pinned = true - tweet.user = result.user - result.pinned = some tweet - proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = if query.fromUser.len != 1: diff --git a/src/tokens.nim b/src/tokens.nim index b69786e..decf228 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -44,10 +44,10 @@ proc getPoolJson*(): JsonNode = of Api.search: 100000 of Api.photoRail: 180 of Api.timeline: 187 - of Api.userTweets: 300 + of Api.userTweets, Api.userTimeline: 300 of Api.userTweetsAndReplies, Api.userRestId, - Api.userScreenName, Api.tweetDetail, Api.tweetResult: 500 - of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 + Api.userScreenName, Api.tweetDetail, Api.tweetResult, + Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 of Api.userSearch: 900 reqs = maxReqs - token.apis[api].remaining @@ -161,6 +161,6 @@ proc initTokenPool*(cfg: Config) {.async.} = enableLogging = cfg.enableDebug while true: - if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens: + if tokenPool.countIt(not it.isLimited(Api.userTimeline)) < cfg.minTokens: await poolTokens(min(4, cfg.minTokens - tokenPool.len)) await sleepAsync(2000) diff --git a/src/types.nim b/src/types.nim index 5db9ec3..1a47d25 100644 --- a/src/types.nim +++ b/src/types.nim @@ -18,6 +18,7 @@ type tweetDetail tweetResult timeline + userTimeline photoRail search userSearch From 967f5e50f9c2ba4ac50dfb39fc559e104cedbc99 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 8 Aug 2023 02:15:32 +0200 Subject: [PATCH 06/12] Update and disable some tests --- tests/test_profile.py | 2 +- tests/test_timeline.py | 7 +++---- tests/test_tweet_media.py | 14 +++++++------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_profile.py b/tests/test_profile.py index 4c75ad2..38c5189 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -4,7 +4,7 @@ from parameterized import parameterized profiles = [ ['mobile_test', 'Test account', 'Test Account. test test Testing username with @mobile_test_2 and a #hashtag', - 'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '100'], + 'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '98'], ['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13'] ] diff --git a/tests/test_timeline.py b/tests/test_timeline.py index a630ae1..90eaa28 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -13,10 +13,9 @@ empty = [['emptyuser'], ['mobile_test_10']] protected = [['mobile_test_7'], ['Empty_user']] photo_rail = [['mobile_test', [ - 'BzUnaDFCUAAmrjs', 'Bo0nDsYIYAIjqVn', 'Bos--KNIQAAA7Li', 'Boq1sDJIYAAxaoi', - 'BonISmPIEAAhP3G', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG', - 'Bn8QIG3IYAA0IGT', 'Bn8O3QeIUAAONai', 'Bn8NGViIAAATNG4', 'BkKovdrCUAAEz79', - 'BkKoe_oCIAASAqr', 'BkKoRLNCAAAYfDf', 'BkKndxoCQAE1vFt', 'BPEmIbYCMAE44dl' + 'Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG', + 'Bn8QIG3IYAA0IGT', 'Bn8O3QeIUAAONai', 'Bn8NGViIAAATNG4', 'BkKoRLNCAAAYfDf', + 'BkKndxoCQAE1vFt' ]]] diff --git a/tests/test_tweet_media.py b/tests/test_tweet_media.py index 233990e..7a00983 100644 --- a/tests/test_tweet_media.py +++ b/tests/test_tweet_media.py @@ -28,14 +28,14 @@ video_m3u8 = [ ] gallery = [ - ['mobile_test/status/451108446603980803', [ - ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO'] - ]], + # ['mobile_test/status/451108446603980803', [ + # ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO'] + # ]], - ['mobile_test/status/471539824713691137', [ - ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'], - ['Bos--IqIQAAav23'] - ]], + # ['mobile_test/status/471539824713691137', [ + # ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'], + # ['Bos--IqIQAAav23'] + # ]], ['mobile_test/status/469530783384743936', [ ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'], From 54e6ce14ac48409c0552b96e1dadf674c1926c83 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 8 Aug 2023 02:35:43 +0200 Subject: [PATCH 07/12] Simplify photo rail test --- tests/test_timeline.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 90eaa28..9261b44 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -12,11 +12,7 @@ empty = [['emptyuser'], ['mobile_test_10']] protected = [['mobile_test_7'], ['Empty_user']] -photo_rail = [['mobile_test', [ - 'Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG', - 'Bn8QIG3IYAA0IGT', 'Bn8O3QeIUAAONai', 'Bn8NGViIAAATNG4', 'BkKoRLNCAAAYfDf', - 'BkKndxoCQAE1vFt' -]]] +photo_rail = [['mobile_test', ['Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG']]] class TweetTest(BaseTestCase): From d7ca353a55ea3440a2ec1f09155951210a374cc7 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 8 Aug 2023 02:49:58 +0200 Subject: [PATCH 08/12] Disable photo rail test --- tests/test_timeline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 9261b44..b56d6ad 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -55,10 +55,10 @@ class TweetTest(BaseTestCase): self.assert_element_absent(Timeline.older) self.assert_element_absent(Timeline.end) - @parameterized.expand(photo_rail) - def test_photo_rail(self, username, images): - self.open_nitter(username) - self.assert_element_visible(Timeline.photo_rail) - for i, url in enumerate(images): - img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src') - self.assertIn(url, img) + #@parameterized.expand(photo_rail) + #def test_photo_rail(self, username, images): + #self.open_nitter(username) + #self.assert_element_visible(Timeline.photo_rail) + #for i, url in enumerate(images): + #img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src') + #self.assertIn(url, img) From 3572dd77719f549d18aa0872b04564ede1091ca3 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 19 Aug 2023 00:25:14 +0200 Subject: [PATCH 09/12] Replace tokens with guest accounts, swap endpoints --- nitter.nimble | 2 +- src/api.nim | 68 ++++---------- src/apiutils.nim | 54 +++++++---- src/consts.nim | 8 +- src/nitter.nim | 13 ++- src/parser.nim | 193 ++++------------------------------------ src/redis_cache.nim | 18 ++-- src/routes/rss.nim | 4 +- src/routes/search.nim | 4 +- src/routes/timeline.nim | 8 +- src/tokens.nim | 158 +++++++++++--------------------- src/types.nim | 11 ++- 12 files changed, 159 insertions(+), 382 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index 7771b31..e6a1909 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -23,7 +23,7 @@ requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#ca5989a" requires "flatty#e668085" requires "jsony#ea811be" - +requires "oauth#b8c163b" # Tasks diff --git a/src/api.nim b/src/api.nim index c313aa2..d6a4564 100644 --- a/src/api.nim +++ b/src/api.nim @@ -33,23 +33,6 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi js = await fetch(url ? params, apiId) result = parseGraphTimeline(js, "user", after) -# proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} = -# if id.len == 0: return -# let -# ps = genParams({"userId": id, "include_tweet_replies": $replies}, after) -# url = oldUserTweets / (id & ".json") ? ps -# result = parseTimeline(await fetch(url, Api.timeline), after) - -proc getUserTimeline*(id: string; after=""): Future[Profile] {.async.} = - var ps = genParams({"id": id}) - if after.len > 0: - ps.add ("down_cursor", after) - - let - url = legacyUserTweets ? ps - js = await fetch(url, Api.userTimeline) - result = parseUserTimeline(js, after) - proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let @@ -112,10 +95,10 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} = if after.len > 0: result.replies = await getReplies(id, after) -proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} = +proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = let q = genQueryParam(query) if q.len == 0 or q == emptyQuery: - return Profile(tweets: Timeline(query: query, beginning: true)) + return Timeline(query: query, beginning: true) var variables = %*{ @@ -129,44 +112,29 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} = if after.len > 0: variables["cursor"] = % after let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = Profile(tweets: parseGraphSearch(await fetch(url, Api.search), after)) - result.tweets.query = query - -proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = - var q = genQueryParam(query) - - if q.len == 0 or q == emptyQuery: - return Timeline(query: query, beginning: true) - - if after.len > 0: - q &= " max_id:" & after - - let url = tweetSearch ? genParams({ - "q": q , - "modules": "status", - "result_type": "recent", - }) - - result = parseTweetSearch(await fetch(url, Api.search), after) + result = parseGraphSearch[Tweets](await fetch(url, Api.search), after) result.query = query -proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = +proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} = if query.text.len == 0: return Result[User](query: query, beginning: true) - var url = userSearch ? { - "q": query.text, - "skip_status": "1", - "count": "20", - "page": page - } + var + variables = %*{ + "rawQuery": query.text, + "count": 20, + "product": "People", + "withDownvotePerspective": false, + "withReactionsMetadata": false, + "withReactionsPerspective": false + } + if after.len > 0: + variables["cursor"] = % after + result.beginning = false - result = parseUsers(await fetchRaw(url, Api.userSearch)) + let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} + result = parseGraphSearch[User](await fetch(url, Api.search), after) result.query = query - if page.len == 0: - result.bottom = "2" - elif page.allCharsInSet(Digits): - result.bottom = $(parseInt(page) + 1) proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = if name.len == 0: return diff --git a/src/apiutils.nim b/src/apiutils.nim index 1da971a..d1ecfa3 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,6 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-only -import httpclient, asyncdispatch, options, strutils, uri -import jsony, packedjson, zippy +import httpclient, asyncdispatch, options, strutils, uri, times, math +import jsony, packedjson, zippy, oauth1 import types, tokens, consts, parserutils, http_pool import experimental/types/common @@ -29,12 +29,30 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; else: result &= ("cursor", cursor) -proc genHeaders*(token: Token = nil): HttpHeaders = +proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string = + let + encodedUrl = url.replace(",", "%2C").replace("+", "%20") + params = OAuth1Parameters( + consumerKey: consumerKey, + signatureMethod: "HMAC-SHA1", + timestamp: $int(round(epochTime())), + nonce: "0", + isIncludeVersionToHeader: true, + token: oauthToken + ) + signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret) + + params.signature = percentEncode(signature) + + return getOauth1RequestHeader(params)["authorization"] + +proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders = + let header = getOauthHeader(url, oauthToken, oauthTokenSecret) + result = newHttpHeaders({ "connection": "keep-alive", - "authorization": auth, + "authorization": header, "content-type": "application/json", - "x-guest-token": if token == nil: "" else: token.tok, "x-twitter-active-user": "yes", "authority": "api.twitter.com", "accept-encoding": "gzip", @@ -43,24 +61,24 @@ proc genHeaders*(token: Token = nil): HttpHeaders = "DNT": "1" }) -template updateToken() = +template updateAccount() = if resp.headers.hasKey(rlRemaining): let remaining = parseInt(resp.headers[rlRemaining]) reset = parseInt(resp.headers[rlReset]) - token.setRateLimit(api, remaining, reset) + account.setRateLimit(api, remaining, reset) template fetchImpl(result, fetchBody) {.dirty.} = once: pool = HttpPool() - var token = await getToken(api) - if token.tok.len == 0: + var account = await getGuestAccount(api) + if account.oauthToken.len == 0: raise rateLimitError() try: var resp: AsyncResponse - pool.use(genHeaders(token)): + pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)): template getContent = resp = await c.get($url) result = await resp.body @@ -79,19 +97,19 @@ template fetchImpl(result, fetchBody) {.dirty.} = fetchBody - release(token, used=true) + release(account, used=true) if resp.status == $Http400: raise newException(InternalError, $url) except InternalError as e: raise e except BadClientError as e: - release(token, used=true) + release(account, used=true) raise e except Exception as e: - echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url + echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", account.id, ", url: ", url if "length" notin e.msg and "descriptor" notin e.msg: - release(token, invalid=true) + release(account, invalid=true) raise rateLimitError() proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = @@ -103,12 +121,12 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = echo resp.status, ": ", body, " --- url: ", url result = newJNull() - updateToken() + updateAccount() let error = result.getError if error in {invalidToken, badToken}: echo "fetch error: ", result.getError - release(token, invalid=true) + release(account, invalid=true) raise rateLimitError() proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = @@ -117,11 +135,11 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = echo resp.status, ": ", result, " --- url: ", url result.setLen(0) - updateToken() + updateAccount() if result.startsWith("{\"errors"): let errors = result.fromJson(Errors) if errors in {invalidToken, badToken}: echo "fetch error: ", errors - release(token, invalid=true) + release(account, invalid=true) raise rateLimitError() diff --git a/src/consts.nim b/src/consts.nim index a25f6ea..2cfd1ed 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -2,17 +2,13 @@ import uri, sequtils, strutils const - auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" + consumerKey* = "3nVuSoBZnx6U4vzUxf5w" + consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" api = parseUri("https://api.twitter.com") activate* = $(api / "1.1/guest/activate.json") - legacyUserTweets* = api / "1.1/timeline/user.json" photoRail* = api / "1.1/statuses/media_timeline.json" - userSearch* = api / "1.1/users/search.json" - tweetSearch* = api / "1.1/search/universal.json" - - # oldUserTweets* = api / "2/timeline/profile" graphql = api / "graphql" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" diff --git a/src/nitter.nim b/src/nitter.nim index 25a569d..4a4ec13 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -3,6 +3,7 @@ import asyncdispatch, strformat, logging from net import Port from htmlgen import a from os import getEnv +from json import parseJson import jester @@ -15,8 +16,14 @@ import routes/[ const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" -let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") -let (cfg, fullCfg) = getConfig(configPath) +let + configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") + (cfg, fullCfg) = getConfig(configPath) + + accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") + accounts = parseJson(readFile(accountsPath)) + +initAccountPool(cfg, parseJson(readFile(accountsPath))) if not cfg.enableDebug: # Silence Jester's query warning @@ -38,8 +45,6 @@ waitFor initRedisPool(cfg) stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n" stdout.flushFile -asyncCheck initTokenPool(cfg) - createUnsupportedRouter(cfg) createResolverRouter(cfg) createPrefRouter(cfg) diff --git a/src/parser.nim b/src/parser.nim index c7d8bd1..9262b28 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -29,7 +29,9 @@ proc parseUser(js: JsonNode; id=""): User = result.expandUserEntities(js) proc parseGraphUser(js: JsonNode): User = - let user = ? js{"user_result", "result"} + var user = js{"user_result", "result"} + if user.isNull: + user = ? js{"user_results", "result"} result = parseUser(user{"legacy"}) if "is_blue_verified" in user: @@ -287,169 +289,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.text.removeSuffix(" Learn more.") result.available = false -proc parseLegacyTweet(js: JsonNode): Tweet = - result = parseTweet(js, js{"card"}) - if not result.isNil and result.available: - result.user = parseUser(js{"user"}) - - if result.quote.isSome: - result.quote = some parseLegacyTweet(js{"quoted_status"}) - -proc parseTweetSearch*(js: JsonNode; after=""): Timeline = - result.beginning = after.len == 0 - - if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0: - return - - for item in js{"modules"}: - with tweet, item{"status", "data"}: - let parsed = parseLegacyTweet(tweet) - - if parsed.retweet.isSome: - parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"}) - - result.content.add @[parsed] - - if result.content.len > 0: - result.bottom = $(result.content[^1][0].id - 1) - -proc parseUserTimelineTweet(tweet: JsonNode; users: TableRef[string, User]): Tweet = - result = parseTweet(tweet, tweet{"card"}) - - if result.isNil or not result.available: - return - - with user, tweet{"user"}: - let userId = user{"id_str"}.getStr - if user{"ext_is_blue_verified"}.getBool(false): - users[userId].verified = users[userId].verified or true - result.user = users[userId] - -proc parseUserTimeline*(js: JsonNode; after=""): Profile = - result = Profile(tweets: Timeline(beginning: after.len == 0)) - - if js.kind == JNull or "response" notin js or "twitter_objects" notin js: - return - - var users = newTable[string, User]() - for userId, user in js{"twitter_objects", "users"}: - users[userId] = parseUser(user) - - for entity in js{"response", "timeline"}: - let - tweetId = entity{"tweet", "id"}.getId - isPinned = entity{"tweet", "is_pinned"}.getBool(false) - - with tweet, js{"twitter_objects", "tweets", $tweetId}: - var parsed = parseUserTimelineTweet(tweet, users) - - if not parsed.isNil and parsed.available: - if parsed.quote.isSome: - parsed.quote = some parseUserTimelineTweet(tweet{"quoted_status"}, users) - - if parsed.retweet.isSome: - let retweet = parseUserTimelineTweet(tweet{"retweeted_status"}, users) - if retweet.quote.isSome: - retweet.quote = some parseUserTimelineTweet(tweet{"retweeted_status", "quoted_status"}, users) - parsed.retweet = some retweet - - if isPinned: - parsed.pinned = true - result.pinned = some parsed - else: - result.tweets.content.add parsed - - result.tweets.bottom = js{"response", "cursor", "bottom"}.getStr - -# proc finalizeTweet(global: GlobalObjects; id: string): Tweet = -# let intId = if id.len > 0: parseBiggestInt(id) else: 0 -# result = global.tweets.getOrDefault(id, Tweet(id: intId)) - -# if result.quote.isSome: -# let quote = get(result.quote).id -# if $quote in global.tweets: -# result.quote = some global.tweets[$quote] -# else: -# result.quote = some Tweet() - -# if result.retweet.isSome: -# let rt = get(result.retweet).id -# if $rt in global.tweets: -# result.retweet = some finalizeTweet(global, $rt) -# else: -# result.retweet = some Tweet() - -# proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = -# let pin = js{"pinEntry", "entry", "entryId"}.getStr -# if pin.len == 0: return - -# let id = pin.getId -# if id notin global.tweets: return - -# global.tweets[id].pinned = true -# return finalizeTweet(global, id) - -# proc parseGlobalObjects(js: JsonNode): GlobalObjects = -# result = GlobalObjects() -# let -# tweets = ? js{"globalObjects", "tweets"} -# users = ? js{"globalObjects", "users"} - -# for k, v in users: -# result.users[k] = parseUser(v, k) - -# for k, v in tweets: -# var tweet = parseTweet(v, v{"card"}) -# if tweet.user.id in result.users: -# tweet.user = result.users[tweet.user.id] -# result.tweets[k] = tweet - -# proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) = -# if js.kind != JArray or js.len == 0: -# return - -# for i in js: -# if res.tweets.beginning and i{"pinEntry"}.notNull: -# with pin, parsePin(i, global): -# res.pinned = some pin - -# with r, i{"replaceEntry", "entry"}: -# if "top" in r{"entryId"}.getStr: -# res.tweets.top = r.getCursor -# elif "bottom" in r{"entryId"}.getStr: -# res.tweets.bottom = r.getCursor - -# proc parseTimeline*(js: JsonNode; after=""): Profile = -# result = Profile(tweets: Timeline(beginning: after.len == 0)) -# let global = parseGlobalObjects(? js) - -# let instructions = ? js{"timeline", "instructions"} -# if instructions.len == 0: return - -# result.parseInstructions(global, instructions) - -# var entries: JsonNode -# for i in instructions: -# if "addEntries" in i: -# entries = i{"addEntries", "entries"} - -# for e in ? entries: -# let entry = e{"entryId"}.getStr -# if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry: -# let tweet = finalizeTweet(global, e.getEntryId) -# if not tweet.available: continue -# result.tweets.content.add tweet -# elif "cursor-top" in entry: -# result.tweets.top = e.getCursor -# elif "cursor-bottom" in entry: -# result.tweets.bottom = e.getCursor -# elif entry.startsWith("sq-cursor"): -# with cursor, e{"content", "operation", "cursor"}: -# if cursor{"cursorType"}.getStr == "Bottom": -# result.tweets.bottom = cursor{"value"}.getStr -# else: -# result.tweets.top = cursor{"value"}.getStr - proc parsePhotoRail*(js: JsonNode): PhotoRail = with error, js{"error"}: if error.getStr == "Not authorized.": @@ -597,8 +436,8 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = tweet.id = parseBiggestInt(entryId) result.pinned = some tweet -proc parseGraphSearch*(js: JsonNode; after=""): Timeline = - result = Timeline(beginning: after.len == 0) +proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] = + result = Result[T](beginning: after.len == 0) let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"} if instructions.len == 0: @@ -607,15 +446,21 @@ proc parseGraphSearch*(js: JsonNode; after=""): Timeline = for instruction in instructions: let typ = instruction{"type"}.getStr if typ == "TimelineAddEntries": - for e in instructions[0]{"entries"}: + for e in instruction{"entries"}: let entryId = e{"entryId"}.getStr - if entryId.startsWith("tweet"): - with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: - let tweet = parseGraphTweet(tweetResult) - if not tweet.available: - tweet.id = parseBiggestInt(entryId.getId()) - result.content.add tweet - elif entryId.startsWith("cursor-bottom"): + when T is Tweets: + if entryId.startsWith("tweet"): + with tweetRes, e{"content", "itemContent", "tweet_results", "result"}: + let tweet = parseGraphTweet(tweetRes) + if not tweet.available: + tweet.id = parseBiggestInt(entryId.getId()) + result.content.add tweet + elif T is User: + if entryId.startsWith("user"): + with userRes, e{"content", "itemContent"}: + result.content.add parseGraphUser(userRes) + + if entryId.startsWith("cursor-bottom"): result.bottom = e{"content", "value"}.getStr elif typ == "TimelineReplaceEntry": if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"): diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 89161be..2387a42 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -147,15 +147,15 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} = if result.len > 0 and user.id.len > 0: await all(cacheUserId(result, user.id), cache(user)) -proc getCachedTweet*(id: int64): Future[Tweet] {.async.} = - if id == 0: return - let tweet = await get(id.tweetKey) - if tweet != redisNil: - tweet.deserialize(Tweet) - else: - result = await getGraphTweetResult($id) - if not result.isNil: - await cache(result) +# proc getCachedTweet*(id: int64): Future[Tweet] {.async.} = +# if id == 0: return +# let tweet = await get(id.tweetKey) +# if tweet != redisNil: +# tweet.deserialize(Tweet) +# else: +# result = await getGraphTweetResult($id) +# if not result.isNil: +# await cache(result) proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} = if name.len == 0: return diff --git a/src/routes/rss.nim b/src/routes/rss.nim index d378396..6c77992 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -27,7 +27,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. else: var q = query q.fromUser = names - profile.tweets = await getTweetSearch(q, after) + profile.tweets = await getGraphTweetSearch(q, after) # this is kinda dumb profile.user = User( username: name, @@ -76,7 +76,7 @@ proc createRssRouter*(cfg: Config) = if rss.cursor.len > 0: respRss(rss, "Search") - let tweets = await getTweetSearch(query, cursor) + let tweets = await getGraphTweetSearch(query, cursor) rss.cursor = tweets.bottom rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) diff --git a/src/routes/search.nim b/src/routes/search.nim index c270df5..e9f991d 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -29,13 +29,13 @@ proc createSearchRouter*(cfg: Config) = redirect("/" & q) var users: Result[User] try: - users = await getUserSearch(query, getCursor()) + users = await getGraphUserSearch(query, getCursor()) except InternalError: users = Result[User](beginning: true, query: query) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title) of tweets: let - tweets = await getTweetSearch(query, getCursor()) + tweets = await getGraphTweetSearch(query, getCursor()) rss = "/search/rss?" & genQueryUrl(query) resp renderMain(renderTweetSearch(tweets, prefs, getPath()), request, cfg, prefs, title, rss=rss) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 8d02b68..3568ab7 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -53,10 +53,10 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result = case query.kind - of posts: await getUserTimeline(userId, after) + of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) - else: Profile(tweets: await getTweetSearch(query, after)) + else: Profile(tweets: await getGraphTweetSearch(query, after)) result.user = await user result.photoRail = await rail @@ -67,7 +67,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = if query.fromUser.len != 1: let - timeline = await getTweetSearch(query, after) + timeline = await getGraphTweetSearch(query, after) html = renderTweetSearch(timeline, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) @@ -122,7 +122,7 @@ proc createTimelineRouter*(cfg: Config) = # used for the infinite scroll feature if @"scroll".len > 0: if query.fromUser.len != 1: - var timeline = await getTweetSearch(query, after) + var timeline = await getGraphTweetSearch(query, after) if timeline.content.len == 0: resp Http404 timeline.beginning = true resp $renderTweetSearch(timeline, prefs, getPath()) diff --git a/src/tokens.nim b/src/tokens.nim index decf228..71a7abd 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -1,23 +1,16 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, httpclient, times, sequtils, json, random -import strutils, tables -import types, consts +import asyncdispatch, times, json, random, strutils, tables +import types -const - maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions - maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires - maxAge = 2.hours + 55.minutes # tokens expire after 3 hours - failDelay = initDuration(minutes=30) +# max requests at a time per account to avoid race conditions +const maxConcurrentReqs = 5 var - tokenPool: seq[Token] - lastFailed: Time + accountPool: seq[GuestAccount] enableLogging = false -let headers = newHttpHeaders({"authorization": auth}) - template log(str) = - if enableLogging: echo "[tokens] ", str + if enableLogging: echo "[accounts] ", str proc getPoolJson*(): JsonNode = var @@ -26,141 +19,94 @@ proc getPoolJson*(): JsonNode = totalPending = 0 reqsPerApi: Table[string, int] - for token in tokenPool: - totalPending.inc(token.pending) - list[token.tok] = %*{ + for account in accountPool: + totalPending.inc(account.pending) + list[account.id] = %*{ "apis": newJObject(), - "pending": token.pending, - "init": $token.init, - "lastUse": $token.lastUse + "pending": account.pending, } - for api in token.apis.keys: - list[token.tok]["apis"][$api] = %token.apis[api] + for api in account.apis.keys: + list[account.id]["apis"][$api] = %account.apis[api].remaining let maxReqs = case api - of Api.search: 100000 + of Api.search: 50 of Api.photoRail: 180 - of Api.timeline: 187 - of Api.userTweets, Api.userTimeline: 300 - of Api.userTweetsAndReplies, Api.userRestId, - Api.userScreenName, Api.tweetDetail, Api.tweetResult, - Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 - of Api.userSearch: 900 - reqs = maxReqs - token.apis[api].remaining + of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, + Api.userRestId, Api.userScreenName, + Api.tweetDetail, Api.tweetResult, + Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500 + reqs = maxReqs - account.apis[api].remaining reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs totalReqs.inc(reqs) return %*{ - "amount": tokenPool.len, + "amount": accountPool.len, "requests": totalReqs, "pending": totalPending, "apis": reqsPerApi, - "tokens": list + "accounts": list } proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") -proc fetchToken(): Future[Token] {.async.} = - if getTime() - lastFailed < failDelay: - raise rateLimitError() - - let client = newAsyncHttpClient(headers=headers) - - try: - let - resp = await client.postContent(activate) - tokNode = parseJson(resp)["guest_token"] - tok = tokNode.getStr($(tokNode.getInt)) - time = getTime() - - return Token(tok: tok, init: time, lastUse: time) - except Exception as e: - echo "[tokens] fetching token failed: ", e.msg - if "Try again" notin e.msg: - echo "[tokens] fetching tokens paused, resuming in 30 minutes" - lastFailed = getTime() - finally: - client.close() - -proc expired(token: Token): bool = - let time = getTime() - token.init < time - maxAge or token.lastUse < time - maxLastUse - -proc isLimited(token: Token; api: Api): bool = - if token.isNil or token.expired: +proc isLimited(account: GuestAccount; api: Api): bool = + if account.isNil: return true - if api in token.apis: - let limit = token.apis[api] + if api in account.apis: + let limit = account.apis[api] return (limit.remaining <= 10 and limit.reset > epochTime().int) else: return false -proc isReady(token: Token; api: Api): bool = - not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api)) +proc isReady(account: GuestAccount; api: Api): bool = + not (account.isNil or account.pending > maxConcurrentReqs or account.isLimited(api)) -proc release*(token: Token; used=false; invalid=false) = - if token.isNil: return - if invalid or token.expired: - if invalid: log "discarding invalid token" - elif token.expired: log "discarding expired token" +proc release*(account: GuestAccount; used=false; invalid=false) = + if account.isNil: return + if invalid: + log "discarding invalid account: " & account.id - let idx = tokenPool.find(token) - if idx > -1: tokenPool.delete(idx) + let idx = accountPool.find(account) + if idx > -1: accountPool.delete(idx) elif used: - dec token.pending - token.lastUse = getTime() + dec account.pending -proc getToken*(api: Api): Future[Token] {.async.} = - for i in 0 ..< tokenPool.len: +proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} = + for i in 0 ..< accountPool.len: if result.isReady(api): break release(result) - result = tokenPool.sample() + result = accountPool.sample() - if not result.isReady(api): - release(result) - result = await fetchToken() - log "added new token to pool" - tokenPool.add result - - if not result.isNil: + if not result.isNil and result.isReady(api): inc result.pending else: + log "no accounts available for API: " & $api raise rateLimitError() -proc setRateLimit*(token: Token; api: Api; remaining, reset: int) = +proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = # avoid undefined behavior in race conditions - if api in token.apis: - let limit = token.apis[api] + if api in account.apis: + let limit = account.apis[api] if limit.reset >= reset and limit.remaining < remaining: return + if limit.reset == reset and limit.remaining >= remaining: + account.apis[api].remaining = remaining + return - token.apis[api] = RateLimit(remaining: remaining, reset: reset) + account.apis[api] = RateLimit(remaining: remaining, reset: reset) -proc poolTokens*(amount: int) {.async.} = - var futs: seq[Future[Token]] - for i in 0 ..< amount: - futs.add fetchToken() - - for token in futs: - var newToken: Token - - try: newToken = await token - except: discard - - if not newToken.isNil: - log "added new token to pool" - tokenPool.add newToken - -proc initTokenPool*(cfg: Config) {.async.} = +proc initAccountPool*(cfg: Config; accounts: JsonNode) = enableLogging = cfg.enableDebug - while true: - if tokenPool.countIt(not it.isLimited(Api.userTimeline)) < cfg.minTokens: - await poolTokens(min(4, cfg.minTokens - tokenPool.len)) - await sleepAsync(2000) + for account in accounts: + accountPool.add GuestAccount( + id: account{"user", "id_str"}.getStr, + oauthToken: account{"oauth_token"}.getStr, + oauthSecret: account{"oauth_token_secret"}.getStr, + ) diff --git a/src/types.nim b/src/types.nim index 1a47d25..2a553dd 100644 --- a/src/types.nim +++ b/src/types.nim @@ -17,11 +17,8 @@ type Api* {.pure.} = enum tweetDetail tweetResult - timeline - userTimeline photoRail search - userSearch list listBySlug listMembers @@ -36,9 +33,11 @@ type remaining*: int reset*: int - Token* = ref object - tok*: string - init*: Time + GuestAccount* = ref object + id*: string + oauthToken*: string + oauthSecret*: string + # init*: Time lastUse*: Time pending*: int apis*: Table[Api, RateLimit] From bbd68e684071e8b26fca89587db48f12417a9bf0 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 19 Aug 2023 01:13:36 +0200 Subject: [PATCH 10/12] Filter out account limits that already reset --- src/tokens.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tokens.nim b/src/tokens.nim index 71a7abd..401dc05 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -19,6 +19,8 @@ proc getPoolJson*(): JsonNode = totalPending = 0 reqsPerApi: Table[string, int] + let now = epochTime() + for account in accountPool: totalPending.inc(account.pending) list[account.id] = %*{ @@ -27,6 +29,9 @@ proc getPoolJson*(): JsonNode = } for api in account.apis.keys: + if (now.int - account.apis[api].reset) / 60 > 15: + continue + list[account.id]["apis"][$api] = %account.apis[api].remaining let From 3d8858f0d86d09ce815bc78db417290557f23908 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 20 Aug 2023 11:56:42 +0200 Subject: [PATCH 11/12] Track rate limits, reset after 24 hours --- src/apiutils.nim | 12 +++++++++++- src/tokens.nim | 26 ++++++++++++++++++++------ src/types.nim | 2 ++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index d1ecfa3..54e6777 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import httpclient, asyncdispatch, options, strutils, uri, times, math +import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import jsony, packedjson, zippy, oauth1 import types, tokens, consts, parserutils, http_pool import experimental/types/common @@ -129,6 +129,16 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = release(account, invalid=true) raise rateLimitError() + if body.startsWith("{\"errors"): + let errors = body.fromJson(Errors) + if errors in {invalidToken, badToken}: + echo "fetch error: ", errors + release(account, invalid=true) + raise rateLimitError() + elif errors in {rateLimited}: + account.apis[api].limited = true + echo "rate limited, api: ", $api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id + proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = fetchImpl result: if not (result.startsWith('{') or result.startsWith('[')): diff --git a/src/tokens.nim b/src/tokens.nim index 401dc05..45aa895 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -3,7 +3,9 @@ import asyncdispatch, times, json, random, strutils, tables import types # max requests at a time per account to avoid race conditions -const maxConcurrentReqs = 5 +const + maxConcurrentReqs = 5 + dayInSeconds = 24 * 60 * 60 var accountPool: seq[GuestAccount] @@ -19,7 +21,7 @@ proc getPoolJson*(): JsonNode = totalPending = 0 reqsPerApi: Table[string, int] - let now = epochTime() + let now = epochTime().int for account in accountPool: totalPending.inc(account.pending) @@ -29,10 +31,17 @@ proc getPoolJson*(): JsonNode = } for api in account.apis.keys: - if (now.int - account.apis[api].reset) / 60 > 15: - continue + let obj = %*{} + if account.apis[api].limited: + obj["limited"] = %true - list[account.id]["apis"][$api] = %account.apis[api].remaining + if account.apis[api].reset > now.int: + obj["remaining"] = %account.apis[api].remaining + + list[account.id]["apis"][$api] = obj + + if "remaining" notin obj: + continue let maxReqs = @@ -65,7 +74,12 @@ proc isLimited(account: GuestAccount; api: Api): bool = if api in account.apis: let limit = account.apis[api] - return (limit.remaining <= 10 and limit.reset > epochTime().int) + + if limit.limited and (epochTime().int - limit.limitedAt) > dayInSeconds: + account.apis[api].limited = false + echo "account limit reset, api: ", api, ", id: ", account.id + + return limit.limited or (limit.remaining <= 10 and limit.reset > epochTime().int) else: return false diff --git a/src/types.nim b/src/types.nim index 2a553dd..33d0cda 100644 --- a/src/types.nim +++ b/src/types.nim @@ -32,6 +32,8 @@ type RateLimit* = object remaining*: int reset*: int + limited*: bool + limitedAt*: int GuestAccount* = ref object id*: string From e8b5cbef7b984c527675a4df8fe2c7b4d841b13c Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 20 Aug 2023 12:31:08 +0200 Subject: [PATCH 12/12] Add missing limitedAt assignment --- src/apiutils.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apiutils.nim b/src/apiutils.nim index 54e6777..6e333e7 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -137,6 +137,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = raise rateLimitError() elif errors in {rateLimited}: account.apis[api].limited = true + account.apis[api].limitedAt = epochTime().int echo "rate limited, api: ", $api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =