diff --git a/src/api/list.nim b/src/api/list.nim index 3adf75b..1e7a13b 100644 --- a/src/api/list.nim +++ b/src/api/list.nim @@ -21,10 +21,7 @@ proc getListTimeline*(username, list, agent, after: string; media=true): Future[ if result.content.len == 0: return - let last = result.content[^1] - result.minId = - if last.retweet.isNone: $last.id - else: $(get(last.retweet).id) + result.minId = getLastId(result) proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} = let diff --git a/src/api/media.nim b/src/api/media.nim index 3ff74a0..42b2462 100644 --- a/src/api/media.nim +++ b/src/api/media.nim @@ -123,9 +123,10 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} = if html == nil: return parseCard(get(tweet.card), html) -proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} = +proc getPhotoRail*(username, agent: string; skip=false): Future[seq[GalleryPhoto]] {.async.} = + if skip: return let - headers = genHeaders({"x-requested-with": "XMLHttpRequest"}, agent, base / username) + headers = genHeaders(agent, base / username, xml=true) params = {"for_photo_rail": "true", "oldest_unread_id": "0"} url = base / (timelineMediaUrl % username) ? params html = await fetchHtml(url, headers, jsonKey="items_html") diff --git a/src/api/timeline.nim b/src/api/timeline.nim index e62f8ae..71f802b 100644 --- a/src/api/timeline.nim +++ b/src/api/timeline.nim @@ -18,10 +18,24 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string; if not json.hasKey("items_html"): return let html = parseHtml(json["items_html"].to(string)) - let thread = parseChain(html) + let timeline = parseChain(html) - if media: await getMedia(thread, agent) - result.content = thread.content + if media: await getMedia(timeline, agent) + result.content = timeline.content + +proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} = + var url = base / username + if after.len > 0: + url = url ? {"max_position": after} + + let + headers = genHeaders(agent, base / username, auth=true) + html = await fetchHtml(url, headers) + timeline = parseTimeline(html.select("#timeline > .stream-container"), after) + profile = parseTimelineProfile(html) + + if media: await getMedia(timeline, agent) + result = (profile, timeline) proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} = var params = toSeq({ @@ -39,16 +53,19 @@ proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] result = await finishTimeline(json, Query(), after, agent, media) -proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} = - var url = base / username +proc getMediaTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} = + echo "mediaTimeline" + var params = toSeq({ + "include_available_features": "1", + "include_entities": "1", + "reset_error_state": "false" + }) + if after.len > 0: - url = url ? {"max_position": after} + params.add {"max_position": after} - let - headers = genHeaders(agent, base / username, auth=true) - html = await fetchHtml(url, headers) - timeline = parseTimeline(html.select("#timeline > .stream-container"), after) - profile = parseTimelineProfile(html) + let headers = genHeaders(agent, base / username, xml=true) + let json = await fetchJson(base / (timelineMediaUrl % username) ? params, headers) - if media: await getMedia(timeline, agent) - result = (profile, timeline) + result = await finishTimeline(json, Query(kind: QueryKind.media), after, agent, media) + result.minId = getLastId(result) diff --git a/src/api/utils.nim b/src/api/utils.nim index e6df7e3..ce7e854 100644 --- a/src/api/utils.nim +++ b/src/api/utils.nim @@ -1,6 +1,7 @@ import httpclient, asyncdispatch, htmlparser import strutils, json, xmltree, uri +import ../types import consts proc genHeaders*(headers: openArray[tuple[key: string, val: string]]; @@ -52,3 +53,11 @@ proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = result = parseJson(resp) except: return nil + +proc getLastId*(tweets: Result[Tweet]): string = + if tweets.content.len == 0: return + let last = tweets.content[^1] + if last.retweet.isNone: + $last.id + else: + $(get(last.retweet).id) diff --git a/src/routes/rss.nim b/src/routes/rss.nim index b46a2b2..e7d98a6 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -9,7 +9,7 @@ import ../views/general include "../views/rss.nimf" proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} = - let (profile, timeline, _) = + let (profile, timeline) = await fetchSingleTimeline(name, "", getAgent(), query, media=false) if timeline != nil: diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 90ee05a..fa97a56 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -11,12 +11,8 @@ export router_utils export api, cache, formatters, query, agents export profile, timeline, status -type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto]) - proc fetchSingleTimeline*(name, after, agent: string; query: Query; - media=true): Future[ProfileTimeline] {.async.} = - let railFut = getPhotoRail(name, agent) - + media=true): Future[(Profile, Timeline)] {.async.} = var timeline: Timeline var profile: Profile var cachedProfile = hasCachedProfile(name) @@ -31,13 +27,17 @@ proc fetchSingleTimeline*(name, after, agent: string; query: Query; (profile, timeline) = await getProfileAndTimeline(name, agent, after, media) cache(profile) else: - var timelineFut = getSearch[Tweet](query, after, agent, media) + var timelineFut = + if query.kind == QueryKind.media: + getMediaTimeline(name, after, agent, media) + else: + getSearch[Tweet](query, after, agent, media) if cachedProfile.isNone: profile = await getCachedProfile(name, agent) timeline = await timelineFut if profile.username.len == 0: return - return (profile, timeline, await railFut) + return (profile, timeline) proc fetchMultiTimeline*(names: seq[string]; after, agent: string; query: Query): Future[Timeline] {.async.} = @@ -60,7 +60,10 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; rss: string): Fu names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0) if names.len == 1: - let (p, t, r) = await fetchSingleTimeline(names[0], after, agent, query) + let + rail = getPhotoRail(names[0], agent, skip=(query.kind == media)) + (p, t) = await fetchSingleTimeline(names[0], after, agent, query) + r = await rail if p.username.len == 0: return let pHtml = renderProfile(p, t, r, prefs, getPath()) return renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p),