diff --git a/src/agents.nim b/src/agents.nim new file mode 100644 index 0000000..1fddf29 --- /dev/null +++ b/src/agents.nim @@ -0,0 +1,90 @@ +import random + +const userAgents = [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/38.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 9.0; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/38.0", + "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/40.1", + "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/43.0", + "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/50.0", + "Mozilla/5.0 (X11; Linux x86_64) like Gecko", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) like Gecko", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) like Gecko", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00", + "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00", + "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", + "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16" +] + +proc getAgent*(): string = sample(userAgents) diff --git a/src/api.nim b/src/api.nim index 9106414..572f5bd 100644 --- a/src/api.nim +++ b/src/api.nim @@ -4,7 +4,6 @@ import sequtils, strutils, json, xmltree, uri import types, parser, parserutils, formatters, search const - agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" lang = "en-US,en;q=0.9" auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" @@ -61,7 +60,7 @@ proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = except: return nil -proc getGuestToken(force=false): Future[string] {.async.} = +proc getGuestToken(agent: string; force=false): Future[string] {.async.} = if getTime() - tokenUpdated < tokenLifetime and not force and tokenUses < tokenMaxUses: return guestToken @@ -85,7 +84,7 @@ proc getGuestToken(force=false): Future[string] {.async.} = result = json["guest_token"].to(string) guestToken = result -proc getVideo*(tweet: Tweet; token: string) {.async.} = +proc getVideo*(tweet: Tweet; token, agent: string) {.async.} = if tweet.video.isNone(): return let headers = newHttpHeaders({ @@ -102,8 +101,8 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} = if json == nil: if getTime() - tokenUpdated > initDuration(seconds=1): tokenUpdated = getTime() - discard await getGuestToken(force=true) - await getVideo(tweet, guestToken) + discard await getGuestToken(agent, force=true) + await getVideo(tweet, guestToken, agent) return if tweet.card.isNone: @@ -113,31 +112,31 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} = tweet.video = none(Video) tokenUses.inc -proc getVideos*(thread: Thread; token="") {.async.} = +proc getVideos*(thread: Thread; agent: string; token="") {.async.} = if thread == nil: return var gToken = token if gToken.len == 0: - gToken = await getGuestToken() + gToken = await getGuestToken(agent) var videoFuts: seq[Future[void]] for tweet in thread.tweets.filterIt(it.video.isSome): - videoFuts.add getVideo(tweet, gToken) + videoFuts.add getVideo(tweet, gToken, agent) await all(videoFuts) -proc getConversationVideos*(convo: Conversation) {.async.} = - var token = await getGuestToken() +proc getConversationVideos*(convo: Conversation; agent: string) {.async.} = + var token = await getGuestToken(agent) var futs: seq[Future[void]] - futs.add getVideo(convo.tweet, token) - futs.add convo.replies.mapIt(getVideos(it, token)) - futs.add getVideos(convo.before, token) - futs.add getVideos(convo.after, token) + futs.add getVideo(convo.tweet, token, agent) + futs.add convo.replies.mapIt(getVideos(it, token, agent)) + futs.add getVideos(convo.before, token, agent) + futs.add getVideos(convo.after, token, agent) await all(futs) -proc getPoll*(tweet: Tweet) {.async.} = +proc getPoll*(tweet: Tweet; agent: string) {.async.} = if tweet.poll.isNone(): return let headers = newHttpHeaders({ @@ -154,20 +153,20 @@ proc getPoll*(tweet: Tweet) {.async.} = tweet.poll = some(parsePoll(html)) -proc getPolls*(thread: Thread) {.async.} = +proc getPolls*(thread: Thread; agent: string) {.async.} = if thread == nil: return var polls = thread.tweets.filterIt(it.poll.isSome) - await all(polls.map(getPoll)) + await all(polls.mapIt(getPoll(it, agent))) -proc getConversationPolls*(convo: Conversation) {.async.} = +proc getConversationPolls*(convo: Conversation; agent: string) {.async.} = var futs: seq[Future[void]] - futs.add getPoll(convo.tweet) - futs.add getPolls(convo.before) - futs.add getPolls(convo.after) - futs.add convo.replies.map(getPolls) + futs.add getPoll(convo.tweet, agent) + futs.add getPolls(convo.before, agent) + futs.add getPolls(convo.after, agent) + futs.add convo.replies.mapIt(getPolls(it, agent)) await all(futs) -proc getCard*(tweet: Tweet) {.async.} = +proc getCard*(tweet: Tweet; agent: string) {.async.} = if tweet.card.isNone(): return let headers = newHttpHeaders({ @@ -184,20 +183,20 @@ proc getCard*(tweet: Tweet) {.async.} = parseCard(get(tweet.card), html) -proc getCards*(thread: Thread) {.async.} = +proc getCards*(thread: Thread; agent: string) {.async.} = if thread == nil: return var cards = thread.tweets.filterIt(it.card.isSome) - await all(cards.map(getCard)) + await all(cards.mapIt(getCard(it, agent))) -proc getConversationCards*(convo: Conversation) {.async.} = +proc getConversationCards*(convo: Conversation; agent: string) {.async.} = var futs: seq[Future[void]] - futs.add getCard(convo.tweet) - futs.add getCards(convo.before) - futs.add getCards(convo.after) - futs.add convo.replies.map(getCards) + futs.add getCard(convo.tweet, agent) + futs.add getCards(convo.before, agent) + futs.add getCards(convo.after, agent) + futs.add convo.replies.mapIt(getCards(it, agent)) await all(futs) -proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} = +proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} = let headers = newHttpHeaders({ "Accept": jsonAccept, "Referer": $(base / username), @@ -222,7 +221,7 @@ proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] result = parseIntentProfile(html) -proc getProfile*(username: string): Future[Profile] {.async.} = +proc getProfile*(username, agent: string): Future[Profile] {.async.} = let headers = newHttpHeaders({ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9", "Referer": $(base / username), @@ -248,7 +247,7 @@ proc getProfile*(username: string): Future[Profile] {.async.} = result = parsePopupProfile(html) -proc getTweet*(username, id: string): Future[Conversation] {.async.} = +proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} = let headers = newHttpHeaders({ "Accept": jsonAccept, "Referer": $base, @@ -269,13 +268,13 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} = result = parseConversation(html) let - vidsFut = getConversationVideos(result) - pollFut = getConversationPolls(result) - cardFut = getConversationCards(result) + vidsFut = getConversationVideos(result, agent) + pollFut = getConversationPolls(result, agent) + cardFut = getConversationCards(result, agent) await all(vidsFut, pollFut, cardFut) -proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} = +proc finishTimeline(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} = if json == nil: return Timeline() result = Timeline( @@ -292,14 +291,14 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future let html = parseHtml(json["items_html"].to(string)) thread = parseThread(html) - vidsFut = getVideos(thread) - pollFut = getPolls(thread) - cardFut = getCards(thread) + vidsFut = getVideos(thread, agent) + pollFut = getPolls(thread, agent) + cardFut = getCards(thread, agent) await all(vidsFut, pollFut, cardFut) result.tweets = thread.tweets -proc getTimeline*(username, after: string): Future[Timeline] {.async.} = +proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} = let headers = newHttpHeaders({ "Accept": jsonAccept, "Referer": $(base / username), @@ -320,9 +319,9 @@ proc getTimeline*(username, after: string): Future[Timeline] {.async.} = params.add {"max_position": after} let json = await fetchJson(base / (timelineUrl % username) ? params, headers) - result = await finishTimeline(json, none(Query), after) + result = await finishTimeline(json, none(Query), after, agent) -proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} = +proc getTimelineSearch*(username, after, agent: string; query: Query): Future[Timeline] {.async.} = let queryParam = genQueryParam(query) let queryEncoded = encodeUrl(queryParam, usePlus=false) @@ -347,4 +346,4 @@ proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] } let json = await fetchJson(base / timelineSearchUrl ? params, headers) - result = await finishTimeline(json, some(query), after) + result = await finishTimeline(json, some(query), after, agent) diff --git a/src/cache.nim b/src/cache.nim index 63a05ba..a9a31be 100644 --- a/src/cache.nim +++ b/src/cache.nim @@ -12,18 +12,18 @@ var profileCacheTime = initDuration(minutes=10) proc outdated(profile: Profile): bool = getTime() - profile.updated > profileCacheTime -proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} = +proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} = withDb: try: result.getOne("username = ?", username) doAssert not result.outdated() except AssertionError: - var profile = await getProfile(username) + var profile = await getProfile(username, agent) profile.id = result.id result = profile result.update() except KeyError: - result = await getProfile(username) + result = await getProfile(username, agent) if result.username.len > 0: result.insert() diff --git a/src/nitter.nim b/src/nitter.nim index 3bc617c..d3482fc 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -3,7 +3,7 @@ from net import Port import jester, regex -import api, utils, types, cache, formatters, search, config +import api, utils, types, cache, formatters, search, config, agents import views/[general, profile, status] const configPath {.strdefine.} = "./nitter.conf" @@ -11,15 +11,16 @@ let cfg = getConfig(configPath) proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} = let + agent = getAgent() username = name.strip(chars={'/'}) - profileFut = getCachedProfile(username) - railFut = getPhotoRail(username) + profileFut = getCachedProfile(username, agent) + railFut = getPhotoRail(username, agent) var timelineFut: Future[Timeline] if query.isNone: - timelineFut = getTimeline(username, after) + timelineFut = getTimeline(username, after, agent) else: - timelineFut = getTimelineSearch(username, after, get(query)) + timelineFut = getTimelineSearch(username, after, agent, get(query)) let profile = await profileFut if profile.username.len == 0: @@ -69,7 +70,7 @@ routes: get "/@name/status/@id": cond '.' notin @"name" - let conversation = await getTweet(@"name", @"id") + let conversation = await getTweet(@"name", @"id", getAgent()) if conversation == nil or conversation.tweet.id.len == 0: resp Http404, showError("Tweet not found", cfg.title)