Merge pull request #3 from zedeus/card-support

Card support
This commit is contained in:
Zed 2019-07-16 03:47:12 +02:00 committed by GitHub
commit b10d894a11
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 280 additions and 26 deletions

View file

@ -27,17 +27,15 @@ is on implementing missing features.
- Search (images/videos, hashtags, etc.) - Search (images/videos, hashtags, etc.)
- Custom timeline filter - Custom timeline filter
- Media-only/gallery view
- Nitter link previews - Nitter link previews
- Server configuration - Server configuration
- Caching (waiting for [moigagoo/norm#19](https://github.com/moigagoo/norm/pull/19)) - More caching (waiting for [moigagoo/norm#19](https://github.com/moigagoo/norm/pull/19))
- Twitter "Cards" (link previews)
- Simple account system with customizable feed - Simple account system with customizable feed
- Emoji support (WIP, needs font)
- Video support with hls.js - Video support with hls.js
- Json API endpoints - Json API endpoints
- Themes - Themes
- Nitter logo - Nitter logo
- Emoji support (WIP, uses native font for now)
## Why? ## Why?

View file

@ -237,7 +237,7 @@ nav {
.gallery-row .attachment:last-child, .gallery-row .attachments:last-child, .video-container { .gallery-row .attachment:last-child, .gallery-row .attachments:last-child, .video-container {
margin: 0; margin: 0;
max-height: 500px; max-height: 530px;
} }
.attachments .attachment { .attachments .attachment {
@ -419,7 +419,6 @@ video {
.profile-banner-color { .profile-banner-color {
width: 100%; width: 100%;
padding-bottom: 25%; padding-bottom: 25%;
margin-bottom: 8px;
} }
.profile-card { .profile-card {
@ -882,6 +881,136 @@ video {
margin: 0; margin: 0;
} }
.card {
margin: 5px 0;
}
.card-container {
border-radius: 10px;
border-width: 1px;
border-style: solid;
border-color: #404040;
background-color: #121212;
overflow: hidden;
color: inherit;
display: flex;
text-decoration: none !important;
}
.card-container:hover {
border-color: #808080;
}
.card-container .attachments {
margin: 0;
border-radius: 0;
}
.large .card-container {
display: block;
}
.card-content {
padding: 0.5em;
}
.card-title {
overflow: hidden;
text-overflow: ellipsis;
font-weight: bold;
font-size: 1.15em;
}
.card-description {
margin: 0.3em 0;
}
.card-destination {
color: hsla(240,1%,73%,.9);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
display: block;
}
.card-image-container {
width: 98px;
flex-shrink: 0;
position: relative;
overflow: hidden;
}
.large .card-image-container {
width: unset;
}
.card-image-container:before {
content: "";
display: block;
padding-top: 100%;
}
.large .card-image-container:before {
display: none;
}
.card-image {
position: absolute;
top: 0;
left: 0;
bottom: 0;
right: 0;
background-color: white;
}
.large .card-image {
position: unset;
border-style: solid;
border-color: #404040;
border-width: 0;
border-bottom-width: 1px;
}
.card-image img {
width: 100%;
height: 100%;
display: block;
object-fit: cover;
}
.card-overlay {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
display: flex;
justify-content: center;
align-items: center;
opacity: 0.8;
}
.card-overlay-circle {
border-radius: 50%;
background-color: #404040;
width: 40px;
height: 40px;
align-items: center;
display: flex;
border-width: 5px;
border-color: #d8574d;
border-style: solid;
}
.card-overlay-triangle {
width: 0;
height: 0;
border-style: solid;
border-width: 12px 0 12px 17px;
border-color: transparent transparent transparent #d8574d;
margin-left: 14px;
}
.poll-meter { .poll-meter {
overflow: hidden; overflow: hidden;
position: relative; position: relative;

View file

@ -106,7 +106,11 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
await getVideo(tweet, guestToken) await getVideo(tweet, guestToken)
return return
if tweet.card.isNone:
tweet.video = some(parseVideo(json)) tweet.video = some(parseVideo(json))
else:
get(tweet.card).video = some(parseVideo(json))
tweet.video = none(Video)
tokenUses.inc tokenUses.inc
proc getVideos*(thread: Thread; token="") {.async.} = proc getVideos*(thread: Thread; token="") {.async.} =
@ -163,6 +167,36 @@ proc getConversationPolls*(convo: Conversation) {.async.} =
futs.add convo.replies.map(getPolls) futs.add convo.replies.map(getPolls)
await all(futs) await all(futs)
proc getCard*(tweet: Tweet) {.async.} =
if tweet.card.isNone(): return
let headers = newHttpHeaders({
"Accept": cardAccept,
"Referer": $(base / getLink(tweet)),
"User-Agent": agent,
"Authority": "twitter.com",
"Accept-Language": lang,
})
let query = get(tweet.card).query.replace("sensitive=true", "sensitive=false")
let html = await fetchHtml(base / query, headers)
if html == nil: return
parseCard(get(tweet.card), html)
proc getCards*(thread: Thread) {.async.} =
if thread == nil: return
var cards = thread.tweets.filterIt(it.card.isSome)
await all(cards.map(getCard))
proc getConversationCards*(convo: Conversation) {.async.} =
var futs: seq[Future[void]]
futs.add getCard(convo.tweet)
futs.add getCards(convo.before)
futs.add getCards(convo.after)
futs.add convo.replies.map(getCards)
await all(futs)
proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} = proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} =
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": jsonAccept, "Accept": jsonAccept,
@ -234,9 +268,12 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} =
result = parseConversation(html) result = parseConversation(html)
let vidsFut = getConversationVideos(result) let
let pollFut = getConversationPolls(result) vidsFut = getConversationVideos(result)
await all(vidsFut, pollFut) pollFut = getConversationPolls(result)
cardFut = getConversationCards(result)
await all(vidsFut, pollFut, cardFut)
proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} = proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} =
if json == nil: return Timeline() if json == nil: return Timeline()
@ -257,8 +294,9 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future
thread = parseThread(html) thread = parseThread(html)
vidsFut = getVideos(thread) vidsFut = getVideos(thread)
pollFut = getPolls(thread) pollFut = getPolls(thread)
cardFut = getCards(thread)
await all(vidsFut, pollFut) await all(vidsFut, pollFut, cardFut)
result.tweets = thread.tweets result.tweets = thread.tweets
proc getTimeline*(username, after: string): Future[Timeline] {.async.} = proc getTimeline*(username, after: string): Future[Timeline] {.async.} =

View file

@ -10,7 +10,6 @@ const
emailRegex = re"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)" emailRegex = re"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
usernameRegex = re"(^|[^A-z0-9_?])@([A-z0-9_]+)" usernameRegex = re"(^|[^A-z0-9_?])@([A-z0-9_]+)"
picRegex = re"pic.twitter.com/[^ ]+" picRegex = re"pic.twitter.com/[^ ]+"
cardRegex = re"(https?://)?cards.twitter.com/[^ ]+"
ellipsisRegex = re" ?…" ellipsisRegex = re" ?…"
nbsp = $Rune(0x000A0) nbsp = $Rune(0x000A0)
@ -60,7 +59,6 @@ proc linkifyText*(text: string): string =
proc stripTwitterUrls*(text: string): string = proc stripTwitterUrls*(text: string): string =
result = text result = text
result = result.replace(picRegex, "") result = result.replace(picRegex, "")
result = result.replace(cardRegex, "")
result = result.replace(ellipsisRegex, "") result = result.replace(ellipsisRegex, "")
proc getUserpic*(userpic: string; style=""): string = proc getUserpic*(userpic: string; style=""): string =

View file

@ -75,7 +75,7 @@ proc parseTweet*(node: XmlNode): Tweet =
) )
result.getTweetMedia(tweet) result.getTweetMedia(tweet)
result.getTweetCards(tweet) result.getTweetCard(tweet)
let by = tweet.selectText(".js-retweet-text > a > b") let by = tweet.selectText(".js-retweet-text > a > b")
if by.len > 0: if by.len > 0:
@ -178,3 +178,20 @@ proc parsePhotoRail*(node: XmlNode): seq[GalleryPhoto] =
tweetId: img.attr("data-tweet-id"), tweetId: img.attr("data-tweet-id"),
color: img.attr("background-color").replace("style: ", "") color: img.attr("background-color").replace("style: ", "")
) )
proc parseCard*(card: var Card; node: XmlNode) =
card.title = node.selectText("h2.TwitterCard-title")
card.text = node.selectText("p.tcu-resetMargin")
card.dest = node.selectText("span.SummaryCard-destination")
if card.url.len == 0:
card.url = node.select("a").attr("href")
let image = node.select(".tcu-imageWrapper img")
if image != nil:
# workaround for issue 11713
card.image = some(image.attr("data-src").replace("gname", "g&name"))
if card.kind == liveEvent:
card.text = card.title
card.title = node.selectText(".TwitterCard-attribution--category")

View file

@ -1,4 +1,4 @@
import xmltree, htmlparser, strtabs, strformat, times import xmltree, htmlparser, strtabs, strformat, strutils, times
import regex import regex
import types, formatters, api import types, formatters, api
@ -167,10 +167,36 @@ proc getQuoteMedia*(quote: var Quote; node: XmlNode) =
elif gifBadge != nil: elif gifBadge != nil:
quote.badge = "GIF" quote.badge = "GIF"
proc getTweetCards*(tweet: Tweet; node: XmlNode) = proc getTweetCard*(tweet: Tweet; node: XmlNode) =
if node.attr("data-has-cards") == "false": return if node.attr("data-has-cards") == "false": return
if "poll" in node.attr("data-card2-type"): var cardType = node.attr("data-card2-type")
if ":" in cardType:
cardType = cardType.split(":")[^1]
if "poll" in cardType:
tweet.poll = some(Poll()) tweet.poll = some(Poll())
return
let cardDiv = node.select(".card2 > .js-macaw-cards-iframe-container")
if cardDiv == nil: return
var card = Card(
id: tweet.id,
query: cardDiv.attr("data-src")
)
try:
card.kind = parseEnum[CardKind](cardType)
except ValueError:
card.kind = summary
let cardUrl = cardDiv.attr("data-card-url")
for n in node.selectAll(".tweet-text a"):
if n.attr("href") == cardUrl:
card.url = n.attr("data-expanded-url")
tweet.card = some(card)
proc getMoreReplies*(node: XmlNode): int = proc getMoreReplies*(node: XmlNode): int =
let text = node.innerText().strip() let text = node.innerText().strip()

View file

@ -21,7 +21,7 @@ const
proc initQuery*(filters, includes, excludes, separator: string; name=""): Query = proc initQuery*(filters, includes, excludes, separator: string; name=""): Query =
var sep = separator.strip().toUpper() var sep = separator.strip().toUpper()
Query( Query(
queryType: custom, kind: custom,
filters: filters.split(",").filterIt(it in validFilters), filters: filters.split(",").filterIt(it in validFilters),
includes: includes.split(",").filterIt(it in validFilters), includes: includes.split(",").filterIt(it in validFilters),
excludes: excludes.split(",").filterIt(it in validFilters), excludes: excludes.split(",").filterIt(it in validFilters),
@ -31,7 +31,7 @@ proc initQuery*(filters, includes, excludes, separator: string; name=""): Query
proc getMediaQuery*(name: string): Query = proc getMediaQuery*(name: string): Query =
Query( Query(
queryType: media, kind: media,
filters: @["twimg", "native_video"], filters: @["twimg", "native_video"],
fromUser: name, fromUser: name,
sep: "OR" sep: "OR"
@ -39,7 +39,7 @@ proc getMediaQuery*(name: string): Query =
proc getReplyQuery*(name: string): Query = proc getReplyQuery*(name: string): Query =
Query( Query(
queryType: replies, kind: replies,
includes: @["nativeretweets"], includes: @["nativeretweets"],
fromUser: name fromUser: name
) )
@ -61,8 +61,8 @@ proc genQueryParam*(query: Query): string =
return strip(param & filters.join(&" {query.sep} ")) return strip(param & filters.join(&" {query.sep} "))
proc genQueryUrl*(query: Query): string = proc genQueryUrl*(query: Query): string =
result = &"/{query.queryType}?" result = &"/{query.kind}?"
if query.queryType != custom: return if query.kind != custom: return
var params: seq[string] var params: seq[string]
if query.filters.len > 0: if query.filters.len > 0:

View file

@ -31,11 +31,11 @@ db("cache.db", "", "", ""):
.}: Time .}: Time
type type
QueryType* = enum QueryKind* = enum
replies, media, custom = "search" replies, media, custom = "search"
Query* = object Query* = object
queryType*: QueryType kind*: QueryKind
filters*: seq[string] filters*: seq[string]
includes*: seq[string] includes*: seq[string]
excludes*: seq[string] excludes*: seq[string]
@ -70,6 +70,25 @@ type
status*: string status*: string
leader*: int leader*: int
CardKind* = enum
summary = "summary"
summaryLarge = "summary_large_image"
promoWebsite = "promo_website"
promoVideo = "promo_video_website"
player = "player"
liveEvent = "live_event"
Card* = object
kind*: CardKind
id*: string
query*: string
url*: string
title*: string
dest*: string
text*: string
image*: Option[string]
video*: Option[Video]
Quote* = object Quote* = object
id*: string id*: string
profile*: Profile profile*: Profile
@ -104,6 +123,7 @@ type
stats*: TweetStats stats*: TweetStats
retweet*: Option[Retweet] retweet*: Option[Retweet]
quote*: Option[Quote] quote*: Option[Quote]
card*: Option[Card]
gif*: Option[Gif] gif*: Option[Gif]
video*: Option[Video] video*: Option[Video]
photos*: seq[string] photos*: seq[string]

View file

@ -14,7 +14,7 @@ proc getTabClass(timeline: Timeline; tab: string): string =
if timeline.query.isNone: if timeline.query.isNone:
if tab == "tweets": if tab == "tweets":
classes.add "active" classes.add "active"
elif $timeline.query.get().queryType == tab: elif $timeline.query.get().kind == tab:
classes.add "active" classes.add "active"
return classes.join(" ") return classes.join(" ")

View file

@ -77,6 +77,32 @@ proc renderPoll(poll: Poll): VNode =
span(class="poll-info"): span(class="poll-info"):
text $poll.votes & " votes • " & poll.status text $poll.votes & " votes • " & poll.status
proc renderCardImage(card: Card): VNode =
buildHtml(tdiv(class="card-image-container")):
tdiv(class="card-image"):
img(src=get(card.image).getSigUrl("pic"))
if card.kind == player:
tdiv(class="card-overlay"):
tdiv(class="card-overlay-circle"):
span(class="card-overlay-triangle")
proc renderCard(card: Card): VNode =
const largeCards = {summaryLarge, liveEvent, promoWebsite, promoVideo}
let large = if card.kind in largeCards: " large" else: ""
buildHtml(tdiv(class=("card" & large))):
a(class="card-container", href=card.url):
if card.image.isSome:
renderCardImage(card)
elif card.video.isSome:
renderVideo(get(card.video))
tdiv(class="card-content-container"):
tdiv(class="card-content"):
h2(class="card-title"): text card.title
p(class="card-description"): text card.text
span(class="card-destination"): text card.dest
proc renderStats(stats: TweetStats): VNode = proc renderStats(stats: TweetStats): VNode =
buildHtml(tdiv(class="tweet-stats")): buildHtml(tdiv(class="tweet-stats")):
span(class="tweet-stat"): text "💬 " & $stats.replies span(class="tweet-stat"): text "💬 " & $stats.replies
@ -160,7 +186,9 @@ proc renderTweet*(tweet: Tweet; class=""; index=0; total=(-1); last=false): VNod
if tweet.quote.isSome: if tweet.quote.isSome:
renderQuote(tweet.quote.get()) renderQuote(tweet.quote.get())
if tweet.photos.len > 0: if tweet.card.isSome:
renderCard(tweet.card.get())
elif tweet.photos.len > 0:
renderAlbum(tweet) renderAlbum(tweet)
elif tweet.video.isSome: elif tweet.video.isSome:
renderVideo(tweet.video.get()) renderVideo(tweet.video.get())