Add support for polls

README.md

@@ -27,18 +27,17 @@ is on implementing missing features.
 
 - "Show Thread" button
 - Twitter "Cards" (link previews)
-- Nitter link previews
 - Search (+ hashtag search)
+- Hiding retweets, showing replies, etc.
 - Emoji support (WIP, needs font)
-- Twitter polls
+- Nitter link previews
 - Server configuration
+- Caching (waiting for [moigagoo/norm#19](https://github.com/moigagoo/norm/pull/19))
 - Simple account system with feed (excludes retweets)
-- Hiding retweets from timelines
+- Media-only/gallery view
 - Video support with hls.js
-- Media-only view
-- Themes
-- File caching
 - Json API endpoints
+- Themes
 - Nitter logo
 
 ## Why?

public/style.css

@@ -742,3 +742,39 @@ video {
     padding-bottom: 5px;
     margin: 0;
 }
+
+.poll-meter {
+    overflow: hidden;
+    position: relative;
+    margin: 6px 0;
+    height: 26px;
+    background: #0f0f0f;
+    border-radius: 5px;
+    display: flex;
+    align-items: center;
+}
+
+.poll-choice-bar {
+    height: 100%;
+    position: absolute;
+    background: #383838;
+}
+
+.leader .poll-choice-bar {
+    background: #8a3731;
+}
+
+.poll-choice-value {
+    position: relative;
+    font-weight: bold;
+    margin-left: 5px;
+    margin-right: 6px;
+}
+
+.poll-choice-option {
+    position: relative;
+}
+
+.poll-info {
+    color: #868687;
+}

src/api.nim

@@ -6,7 +6,9 @@ import ./types, ./parser, ./parserutils
 
 const
   agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
+  lang = "en-US,en;q=0.9"
   auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+  cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
 
   base = parseUri("https://twitter.com/")
   apiBase = parseUri("https://api.twitter.com/1.1/")
@@ -19,6 +21,8 @@ const
   tweetUrl = "status"
   videoUrl = "videos/tweet/config/$1.json"
   tokenUrl = "guest/activate.json"
+  cardUrl = "i/cards/tfw/v1/$1"
+  pollUrl = cardUrl & "?cardname=poll2choice_text_only&lang=en"
 
 var
   guestToken = ""
@@ -75,7 +79,7 @@ proc getGuestToken(force=false): Future[string] {.async.} =
   newClient()
 
   let
-    url = apibase / tokenUrl
+    url = apiBase / tokenUrl
     json = parseJson(await client.postContent($url))
 
   result = json["guest_token"].to(string)
@@ -86,7 +90,7 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
 
   let headers = newHttpHeaders({
     "Accept": "application/json, text/javascript, */*; q=0.01",
-    "Referer": tweet.link,
+    "Referer": $(base / tweet.link),
     "User-Agent": agent,
     "Authorization": auth,
     "x-guest-token": token
@@ -129,11 +133,38 @@ proc getConversationVideos*(convo: Conversation) {.async.} =
 
   await all(futs)
 
-proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
+proc getPoll*(tweet: Tweet) {.async.} =
-  let
+  if tweet.poll.isNone(): return
-    url = base / profileIntentUrl ? {"screen_name": username}
-    html = await fetchHtml(url, headers)
 
+  let headers = newHttpHeaders({
+    "Accept": cardAccept,
+    "Referer": $(base / tweet.link),
+    "User-Agent": agent,
+    "Authority": "twitter.com",
+    "Accept-Language": lang,
+  })
+
+  let url = base / (pollUrl % tweet.id)
+  let html = await fetchHtml(url, headers)
+  if html == nil: return
+
+  tweet.poll = some(parsePoll(html))
+
+proc getPolls*(tweets: Tweets) {.async.} =
+  var polls = tweets.filterIt(it.poll.isSome)
+  await all(polls.map(getPoll))
+
+proc getConversationPolls*(convo: Conversation) {.async.} =
+  var futs: seq[Future[void]]
+  futs.add getPoll(convo.tweet)
+  futs.add getPolls(convo.before)
+  futs.add getPolls(convo.after)
+  futs.add convo.replies.map(getPolls)
+  await all(futs)
+
+proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
+  let url = base / profileIntentUrl ? {"screen_name": username}
+  let html = await fetchHtml(url, headers)
   if html == nil: return Profile()
 
   result = parseIntentProfile(html)
@@ -145,7 +176,7 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
     "User-Agent": agent,
     "X-Twitter-Active-User": "yes",
     "X-Requested-With": "XMLHttpRequest",
-    "Accept-Language": "en-US,en;q=0.9"
+    "Accept-Language": lang
   })
 
   let
@@ -171,7 +202,7 @@ proc getTimeline*(username: string; after=""): Future[Timeline] {.async.} =
     "User-Agent": agent,
     "X-Twitter-Active-User": "yes",
     "X-Requested-With": "XMLHttpRequest",
-    "Accept-Language": "en-US,en;q=0.9"
+    "Accept-Language": lang
   })
 
   var url = timelineUrl % username
@@ -194,7 +225,10 @@ proc getTimeline*(username: string; after=""): Future[Timeline] {.async.} =
   let html = parseHtml(json["items_html"].to(string))
 
   result.tweets = parseTweets(html)
-  await getVideos(result.tweets)
+
+  let vidsFut = getVideos(result.tweets)
+  let pollFut = getPolls(result.tweets)
+  await all(vidsFut, pollFut)
 
 proc getTweet*(username: string; id: string): Future[Conversation] {.async.} =
   let headers = newHttpHeaders({
@@ -203,7 +237,7 @@ proc getTweet*(username: string; id: string): Future[Conversation] {.async.} =
     "User-Agent": agent,
     "X-Twitter-Active-User": "yes",
     "X-Requested-With": "XMLHttpRequest",
-    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Language": lang,
     "pragma": "no-cache",
     "x-previous-page-name": "profile"
   })
@@ -215,4 +249,7 @@ proc getTweet*(username: string; id: string): Future[Conversation] {.async.} =
   if html == nil: return
 
   result = parseConversation(html)
-  await getConversationVideos(result)
+
+  let vidsFut = getConversationVideos(result)
+  let pollFut = getConversationPolls(result)
+  await all(vidsFut, pollFut)

src/parser.nim

@@ -71,6 +71,7 @@ proc parseTweet*(node: XmlNode): Tweet =
 
   result.getTweetStats(tweet)
   result.getTweetMedia(tweet)
+  result.getTweetCards(tweet)
 
   let by = tweet.selectText(".js-retweet-text > a > b")
   if by.len > 0:
@@ -136,3 +137,24 @@ proc parseVideo*(node: JsonNode): Video =
     echo "Can't parse video of type ", cType
 
   result.thumb = node["posterImage"].to(string)
+
+proc parsePoll*(node: XmlNode): Poll =
+  let
+    choices = node.selectAll(".PollXChoice-choice")
+    votes = node.selectText(".PollXChoice-footer--total")
+
+  result.votes = votes.strip().split(" ")[0]
+  result.status = node.selectText(".PollXChoice-footer--time")
+
+  for choice in choices:
+    for span in choice.select(".PollXChoice-choice--text").filterIt(it.kind != xnText):
+      if span.attr("class").len == 0:
+        result.options.add span.innerText()
+      elif "progress" in span.attr("class"):
+        result.values.add parseInt(span.innerText()[0 .. ^2])
+
+  var highest = 0
+  for i, n in result.values:
+    if n > highest:
+      highest = n
+      result.leader = i

src/parserutils.nim

@@ -157,3 +157,8 @@ proc getQuoteMedia*(quote: var Quote; node: XmlNode) =
     quote.badge = some(badge.innerText())
   elif gifBadge != nil:
     quote.badge = some("GIF")
+
+proc getTweetCards*(tweet: Tweet; node: XmlNode) =
+  if node.attr("data-has-cards") == "false": return
+  if "poll" in node.attr("data-card2-type"):
+    tweet.poll = some(Poll())

src/types.nim

@@ -47,6 +47,13 @@ type
     url*: string
     thumb*: string
 
+  Poll* = object
+    options*: seq[string]
+    values*: seq[int]
+    votes*: string
+    status*: string
+    leader*: int
+
   Quote* = object
     id*: string
     profile*: Profile
@@ -73,6 +80,7 @@ type
     gif*: Option[Gif]
     video*: Option[Video]
     photos*: seq[string]
+    poll*: Option[Poll]
     available*: bool
 
   Tweets* = seq[Tweet]

src/views/tweet.nimf

@@ -118,6 +118,20 @@
 </div>
 #end proc
 #
+#proc renderPoll(poll: Poll): string =
+<div class="poll">
+  #for i in 0 ..< poll.options.len:
+  #let leader = if poll.leader == i: " leader" else: ""
+  <div class="poll-meter${leader}">
+    <span class="poll-choice-bar" style="width: ${poll.values[i]}%"></span>
+    <span class="poll-choice-value">${poll.values[i]}%</span>
+    <span class="poll-choice-option">${poll.options[i]}</span>
+  </div>
+  #end for
+  <span class="poll-info">${poll.votes} votes • ${poll.status}</span>
+</div>
+#end proc
+#
 #proc renderStats(tweet: Tweet): string =
 <div class="tweet-stats">
   <span class="tweet-stat">💬 ${$tweet.replies}</span>
@@ -146,6 +160,8 @@
     ${renderGif(tweet.gif.get())}
     #elif tweet.quote.isSome:
     ${renderQuote(tweet.quote.get())}
+    #elif tweet.poll.isSome:
+     ${renderPoll(tweet.poll.get())}
     #end if
     ${renderStats(tweet)}
   </div>