From ffce6e21ab54c30dbbee4c2a0e305822e69e99ed Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 23 Oct 2019 08:34:03 +0200 Subject: [PATCH] Use media endpoint for profile media tab This bypasses "search" rate limits. It now includes media beyond images and videos (eg. YouTube links are "media"), but the old behaviour can be restored by clicking search, then filtering "Media" and excluding retweets and replies. --- src/api/list.nim | 5 +---- src/api/media.nim | 5 +++-- src/api/timeline.nim | 43 ++++++++++++++++++++++++++++------------- src/api/utils.nim | 9 +++++++++ src/routes/rss.nim | 2 +- src/routes/timeline.nim | 19 ++++++++++-------- 6 files changed, 55 insertions(+), 28 deletions(-) diff --git a/src/api/list.nim b/src/api/list.nim index 3adf75b..1e7a13b 100644 --- a/src/api/list.nim +++ b/src/api/list.nim @@ -21,10 +21,7 @@ proc getListTimeline*(username, list, agent, after: string; media=true): Future[ if result.content.len == 0: return - let last = result.content[^1] - result.minId = - if last.retweet.isNone: $last.id - else: $(get(last.retweet).id) + result.minId = getLastId(result) proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} = let diff --git a/src/api/media.nim b/src/api/media.nim index 3ff74a0..42b2462 100644 --- a/src/api/media.nim +++ b/src/api/media.nim @@ -123,9 +123,10 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} = if html == nil: return parseCard(get(tweet.card), html) -proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} = +proc getPhotoRail*(username, agent: string; skip=false): Future[seq[GalleryPhoto]] {.async.} = + if skip: return let - headers = genHeaders({"x-requested-with": "XMLHttpRequest"}, agent, base / username) + headers = genHeaders(agent, base / username, xml=true) params = {"for_photo_rail": "true", "oldest_unread_id": "0"} url = base / (timelineMediaUrl % username) ? params html = await fetchHtml(url, headers, jsonKey="items_html") diff --git a/src/api/timeline.nim b/src/api/timeline.nim index e62f8ae..71f802b 100644 --- a/src/api/timeline.nim +++ b/src/api/timeline.nim @@ -18,10 +18,24 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string; if not json.hasKey("items_html"): return let html = parseHtml(json["items_html"].to(string)) - let thread = parseChain(html) + let timeline = parseChain(html) - if media: await getMedia(thread, agent) - result.content = thread.content + if media: await getMedia(timeline, agent) + result.content = timeline.content + +proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} = + var url = base / username + if after.len > 0: + url = url ? {"max_position": after} + + let + headers = genHeaders(agent, base / username, auth=true) + html = await fetchHtml(url, headers) + timeline = parseTimeline(html.select("#timeline > .stream-container"), after) + profile = parseTimelineProfile(html) + + if media: await getMedia(timeline, agent) + result = (profile, timeline) proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} = var params = toSeq({ @@ -39,16 +53,19 @@ proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] result = await finishTimeline(json, Query(), after, agent, media) -proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} = - var url = base / username +proc getMediaTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} = + echo "mediaTimeline" + var params = toSeq({ + "include_available_features": "1", + "include_entities": "1", + "reset_error_state": "false" + }) + if after.len > 0: - url = url ? {"max_position": after} + params.add {"max_position": after} - let - headers = genHeaders(agent, base / username, auth=true) - html = await fetchHtml(url, headers) - timeline = parseTimeline(html.select("#timeline > .stream-container"), after) - profile = parseTimelineProfile(html) + let headers = genHeaders(agent, base / username, xml=true) + let json = await fetchJson(base / (timelineMediaUrl % username) ? params, headers) - if media: await getMedia(timeline, agent) - result = (profile, timeline) + result = await finishTimeline(json, Query(kind: QueryKind.media), after, agent, media) + result.minId = getLastId(result) diff --git a/src/api/utils.nim b/src/api/utils.nim index e6df7e3..ce7e854 100644 --- a/src/api/utils.nim +++ b/src/api/utils.nim @@ -1,6 +1,7 @@ import httpclient, asyncdispatch, htmlparser import strutils, json, xmltree, uri +import ../types import consts proc genHeaders*(headers: openArray[tuple[key: string, val: string]]; @@ -52,3 +53,11 @@ proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = result = parseJson(resp) except: return nil + +proc getLastId*(tweets: Result[Tweet]): string = + if tweets.content.len == 0: return + let last = tweets.content[^1] + if last.retweet.isNone: + $last.id + else: + $(get(last.retweet).id) diff --git a/src/routes/rss.nim b/src/routes/rss.nim index b46a2b2..e7d98a6 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -9,7 +9,7 @@ import ../views/general include "../views/rss.nimf" proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} = - let (profile, timeline, _) = + let (profile, timeline) = await fetchSingleTimeline(name, "", getAgent(), query, media=false) if timeline != nil: diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 90ee05a..fa97a56 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -11,12 +11,8 @@ export router_utils export api, cache, formatters, query, agents export profile, timeline, status -type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto]) - proc fetchSingleTimeline*(name, after, agent: string; query: Query; - media=true): Future[ProfileTimeline] {.async.} = - let railFut = getPhotoRail(name, agent) - + media=true): Future[(Profile, Timeline)] {.async.} = var timeline: Timeline var profile: Profile var cachedProfile = hasCachedProfile(name) @@ -31,13 +27,17 @@ proc fetchSingleTimeline*(name, after, agent: string; query: Query; (profile, timeline) = await getProfileAndTimeline(name, agent, after, media) cache(profile) else: - var timelineFut = getSearch[Tweet](query, after, agent, media) + var timelineFut = + if query.kind == QueryKind.media: + getMediaTimeline(name, after, agent, media) + else: + getSearch[Tweet](query, after, agent, media) if cachedProfile.isNone: profile = await getCachedProfile(name, agent) timeline = await timelineFut if profile.username.len == 0: return - return (profile, timeline, await railFut) + return (profile, timeline) proc fetchMultiTimeline*(names: seq[string]; after, agent: string; query: Query): Future[Timeline] {.async.} = @@ -60,7 +60,10 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; rss: string): Fu names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0) if names.len == 1: - let (p, t, r) = await fetchSingleTimeline(names[0], after, agent, query) + let + rail = getPhotoRail(names[0], agent, skip=(query.kind == media)) + (p, t) = await fetchSingleTimeline(names[0], after, agent, query) + r = await rail if p.username.len == 0: return let pHtml = renderProfile(p, t, r, prefs, getPath()) return renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p),