From 1213220ef0c9b258ecbdff41fcb12dcee29ab18e Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 24 Jun 2019 05:29:47 +0200 Subject: [PATCH] Cache video token --- src/api.nim | 132 +++++++++++++++++++++++--------------------- src/parserutils.nim | 2 +- 2 files changed, 70 insertions(+), 64 deletions(-) diff --git a/src/api.nim b/src/api.nim index 7db6732..68671c3 100644 --- a/src/api.nim +++ b/src/api.nim @@ -18,12 +18,19 @@ const videoUrl = "videos/tweet/config/$1.json" tokenUrl = "guest/activate.json" -proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} = +var + token = "" + tokenUpdated: Time + tokenLifetime = initDuration(minutes=10) + +template newClient() {.dirty.} = var client = newAsyncHttpClient() defer: client.close() - client.headers = headers +proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} = + newClient() + var resp = "" try: resp = await client.getContent($url) @@ -37,10 +44,7 @@ proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {. return parseHtml(resp) proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = - var client = newAsyncHttpClient() - defer: client.close() - - client.headers = headers + newClient() var resp = "" try: @@ -50,6 +54,65 @@ proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = return parseJson(resp) +proc getGuestToken(): Future[string] {.async.} = + if getTime() - tokenUpdated < tokenLifetime: + return token + + tokenUpdated = getTime() + + let headers = newHttpHeaders({ + "Accept": "application/json, text/javascript, */*; q=0.01", + "Referer": $base, + "User-Agent": agent, + "Authorization": auth + }) + + newClient() + + let + url = apibase / tokenUrl + json = parseJson(await client.postContent($url)) + + result = json["guest_token"].to(string) + token = result + +proc getVideo*(tweet: Tweet; token: string) {.async.} = + if not tweet.video.isSome: return + + let headers = newHttpHeaders({ + "Accept": "application/json, text/javascript, */*; q=0.01", + "Referer": tweet.link, + "User-Agent": agent, + "Authorization": auth, + "x-guest-token": token + }) + + let + url = apiBase / (videoUrl % tweet.id) + json = await fetchJson(url, headers) + + tweet.video = some(parseVideo(json)) + +proc getVideos*(tweets: Tweets) {.async.} = + var token = await getGuestToken() + var videoFuts: seq[Future[void]] + + for tweet in tweets.filterIt(it.video.isSome): + videoFuts.add getVideo(tweet, token) + + await all(videoFuts) + +proc getConversationVideos*(convo: Conversation) {.async.} = + var token = await getGuestToken() + var futs: seq[Future[void]] + + futs.add getVideo(convo.tweet, token) + futs.add getVideos(convo.before) + futs.add getVideos(convo.after) + futs.add convo.replies.mapIt(getVideos(it)) + + await all(futs) + proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} = let url = base / profileIntentUrl ? {"screen_name": username} @@ -81,63 +144,6 @@ proc getProfile*(username: string): Future[Profile] {.async.} = result = parsePopupProfile(html) -proc getGuestToken(): Future[string] {.async.} = - let headers = newHttpHeaders({ - "Accept": "application/json, text/javascript, */*; q=0.01", - "Referer": $base, - "User-Agent": agent, - "Authorization": auth - }) - - let client = newAsyncHttpClient() - client.headers = headers - - let - url = apibase / tokenUrl - json = parseJson(await client.postContent($url)) - - result = json["guest_token"].to(string) - -proc getVideo*(tweet: Tweet; token: string) {.async.} = - let headers = newHttpHeaders({ - "Accept": "application/json, text/javascript, */*; q=0.01", - "Referer": tweet.link, - "User-Agent": agent, - "Authorization": auth, - "x-guest-token": token - }) - - let - url = apiBase / (videoUrl % tweet.id) - json = await fetchJson(url, headers) - - - tweet.video = some(parseVideo(json)) - -proc getVideos*(tweets: Tweets; token="") {.async.} = - if not tweets.anyIt(it.video.isSome): return - - var - token = if token.len > 0: token else: await getGuestToken() - videoFuts: seq[Future[void]] - - for tweet in tweets: - if tweet.video.isSome: - videoFuts.add getVideo(tweet, token) - - await all(videoFuts) - -proc getConversationVideos*(convo: Conversation) {.async.} = - var token = await getGuestToken() - var futs: seq[Future[void]] - - futs.add getVideo(convo.tweet, token) - futs.add getVideos(convo.before, token=token) - futs.add getVideos(convo.after, token=token) - futs.add convo.replies.mapIt(getVideos(it, token=token)) - - await all(futs) - proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} = let headers = newHttpHeaders({ "Accept": "application/json, text/javascript, */*; q=0.01", diff --git a/src/parserutils.nim b/src/parserutils.nim index c81d9ac..7fe11a5 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -112,5 +112,5 @@ proc getTweetMedia*(tweet: Tweet; node: XmlNode) = if "gif" in player.getAttr("class"): tweet.gif = some(getGif(player.querySelector(".PlayableMedia-player"))) - else: + elif "video" in player.getAttr("class"): tweet.video = some(Video())