Add user search

This commit is contained in:
Zed 2019-09-13 22:24:58 +02:00
parent eeae28da0c
commit 30bab22dae
16 changed files with 209 additions and 64 deletions

View file

@ -1,2 +1,2 @@
import api/[media, profile, timeline, tweet, search] import api/[profile, timeline, tweet, search, media]
export profile, timeline, tweet, search, media export profile, timeline, tweet, search, media

View file

@ -1,32 +1,56 @@
import httpclient, asyncdispatch, htmlparser import httpclient, asyncdispatch, htmlparser
import sequtils, strutils, json, xmltree, uri import sequtils, strutils, json, xmltree, uri
import ".."/[types, parser, parserutils, formatters, search] import ".."/[types, parser, parserutils, formatters, query]
import utils, consts, media, timeline import utils, consts, media, timeline
proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.async.} = proc getResult[T](json: JsonNode; query: Query; after: string): Result[T] =
let queryParam = genQueryParam(query) Result[T](
let queryEncoded = encodeUrl(queryParam, usePlus=false) hasMore: json["has_more_items"].to(bool),
maxId: json.getOrDefault("max_position").getStr(""),
minId: json.getOrDefault("min_position").getStr("").cleanPos(),
query: query.some,
beginning: after.len == 0
)
let headers = newHttpHeaders({ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
let
kind = if query.kind == users: "users" else: "tweets"
pos = when T is Tweet: genPos(after) else: after
param = genQueryParam(query)
encoded = encodeUrl(param, usePlus=false)
headers = newHttpHeaders({
"Accept": jsonAccept, "Accept": jsonAccept,
"Referer": $(base / ("search?f=tweets&vertical=default&q=$1&src=typd" % queryEncoded)), "Referer": $(base / ("search?f=$1&q=$2&src=typd" % [kind, encoded])),
"User-Agent": agent, "User-Agent": agent,
"X-Requested-With": "XMLHttpRequest", "X-Requested-With": "XMLHttpRequest",
"Authority": "twitter.com", "Authority": "twitter.com",
"Accept-Language": lang "Accept-Language": lang
}) })
let params = { params = {
"f": "tweets", "f": kind,
"vertical": "default", "vertical": "default",
"q": queryParam, "q": param,
"src": "typd", "src": "typd",
"include_available_features": "1", "include_available_features": "1",
"include_entities": "1", "include_entities": "1",
"max_position": if after.len > 0: genPos(after) else: "0", "max_position": if pos.len > 0: pos else: "0",
"reset_error_state": "false" "reset_error_state": "false"
} }
let json = await fetchJson(base / searchUrl ? params, headers) let json = await fetchJson(base / searchUrl ? params, headers)
if json == nil: return Result[T]()
result = getResult[T](json, query, after)
if not json.hasKey("items_html"): return
let html = parseHtml(json["items_html"].to(string))
when T is Tweet:
result = await finishTimeline(json, some(query), after, agent) result = await finishTimeline(json, some(query), after, agent)
elif T is Profile:
result.hasMore = json["items_html"].to(string) != "\n"
for p in html.selectAll(".js-stream-item"):
result.content.add parsePopupProfile(p, ".ProfileCard")

View file

@ -1,7 +1,7 @@
import httpclient, asyncdispatch, htmlparser import httpclient, asyncdispatch, htmlparser
import sequtils, strutils, json, xmltree, uri import sequtils, strutils, json, xmltree, uri
import ".."/[types, parser, parserutils, formatters, search] import ".."/[types, parser, parserutils, formatters, query]
import utils, consts, media import utils, consts, media
proc finishTimeline*(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} = proc finishTimeline*(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =

View file

@ -5,13 +5,14 @@ import jester
import types, config, prefs import types, config, prefs
import views/[general, about] import views/[general, about]
import routes/[preferences, timeline, media, rss] import routes/[preferences, timeline, media, search, rss]
const configPath {.strdefine.} = "./nitter.conf" const configPath {.strdefine.} = "./nitter.conf"
let cfg = getConfig(configPath) let cfg = getConfig(configPath)
createPrefRouter(cfg) createPrefRouter(cfg)
createTimelineRouter(cfg) createTimelineRouter(cfg)
createSearchRouter(cfg)
createMediaRouter(cfg) createMediaRouter(cfg)
createRssRouter(cfg) createRssRouter(cfg)
@ -27,13 +28,9 @@ routes:
get "/about": get "/about":
resp renderMain(renderAbout(), Prefs(), cfg.title) resp renderMain(renderAbout(), Prefs(), cfg.title)
post "/search":
if @"query".len == 0:
resp Http404, showError("Please enter a username.", cfg.title)
redirect("/" & @"query")
extend preferences, "" extend preferences, ""
extend rss, "" extend rss, ""
extend search, ""
extend timeline, "" extend timeline, ""
extend media, "" extend media, ""

View file

@ -23,14 +23,14 @@ proc parseTimelineProfile*(node: XmlNode): Profile =
result.getProfileStats(node.select(".ProfileNav-list")) result.getProfileStats(node.select(".ProfileNav-list"))
proc parsePopupProfile*(node: XmlNode): Profile = proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
let profile = node.select(".profile-card") let profile = node.select(selector)
if profile == nil: return if profile == nil: return
result = Profile( result = Profile(
fullname: profile.getName(".fullname"), fullname: profile.getName(".fullname"),
username: profile.getUsername(".username"), username: profile.getUsername(".username"),
bio: profile.getBio(".bio"), bio: profile.getBio(".bio", fallback=".ProfileCard-bio"),
userpic: profile.getAvatar(".ProfileCard-avatarImage"), userpic: profile.getAvatar(".ProfileCard-avatarImage"),
verified: isVerified(profile), verified: isVerified(profile),
protected: isProtected(profile), protected: isProtected(profile),

View file

@ -86,8 +86,11 @@ proc getName*(profile: XmlNode; selector: string): string =
proc getUsername*(profile: XmlNode; selector: string): string = proc getUsername*(profile: XmlNode; selector: string): string =
profile.selectText(selector).strip(chars={'@', ' ', '\n'}) profile.selectText(selector).strip(chars={'@', ' ', '\n'})
proc getBio*(profile: XmlNode; selector: string): string = proc getBio*(profile: XmlNode; selector: string; fallback=""): string =
profile.selectText(selector).stripText() var bio = profile.selectText(selector)
if bio.len == 0 and fallback.len > 0:
bio = profile.selectText(fallback)
stripText(bio)
proc getAvatar*(profile: XmlNode; selector: string): string = proc getAvatar*(profile: XmlNode; selector: string): string =
profile.selectAttr(selector, "src").getUserpic() profile.selectAttr(selector, "src").getUserpic()

View file

@ -18,8 +18,7 @@ const
posPrefix = "thGAVUV0VFVBa" posPrefix = "thGAVUV0VFVBa"
posSuffix = "EjUAFQAlAFUAFQAA" posSuffix = "EjUAFQAlAFUAFQAA"
proc initQuery*(filters, includes, excludes, separator, text: string; proc initQuery*(filters, includes, excludes, separator, text: string; name=""): Query =
name=""): Query =
var sep = separator.strip().toUpper() var sep = separator.strip().toUpper()
Query( Query(
kind: custom, kind: custom,
@ -50,6 +49,9 @@ proc genQueryParam*(query: Query): string =
var filters: seq[string] var filters: seq[string]
var param: string var param: string
if query.kind == users:
return query.text
for i, user in query.fromUser: for i, user in query.fromUser:
param &= &"from:{user} " param &= &"from:{user} "
if i < query.fromUser.high: if i < query.fromUser.high:
@ -67,12 +69,18 @@ proc genQueryParam*(query: Query): string =
result &= " " & query.text result &= " " & query.text
proc genQueryUrl*(query: Query): string = proc genQueryUrl*(query: Query): string =
if query.kind == multi: return "?" if query.fromUser.len > 0:
result = "/" & query.fromUser.join(",")
result = &"/{query.kind}?" if query.kind == multi:
if query.kind != custom: return return result & "?"
var params: seq[string] if query.kind notin {custom, users}:
return result & &"/{query.kind}?"
result &= &"/search?"
var params = @[&"kind={query.kind}"]
if query.filters.len > 0: if query.filters.len > 0:
params &= "filter=" & query.filters.join(",") params &= "filter=" & query.filters.join(",")
if query.includes.len > 0: if query.includes.len > 0:
@ -84,7 +92,7 @@ proc genQueryUrl*(query: Query): string =
if query.text.len > 0: if query.text.len > 0:
params &= "text=" & query.text params &= "text=" & query.text
if params.len > 0: if params.len > 0:
result &= params.join("&") & "&" result &= params.join("&")
proc cleanPos*(pos: string): string = proc cleanPos*(pos: string): string =
pos.multiReplace((posPrefix, ""), (posSuffix, "")) pos.multiReplace((posPrefix, ""), (posSuffix, ""))

30
src/routes/search.nim Normal file
View file

@ -0,0 +1,30 @@
import strutils, uri
import jester
import router_utils
import ".."/[query, types, utils, api, agents]
import ../views/[general, search]
export search
proc createSearchRouter*(cfg: Config) =
router search:
get "/search":
if @"text".len == 0 or "." in @"text":
resp Http404, showError("Please enter a valid username.", cfg.title)
if "," in @"text":
redirect("/" & @"text")
let query = Query(kind: parseEnum[QueryKind](@"kind", custom), text: @"text")
case query.kind
of users:
let users = await getSearch[Profile](query, @"after", getAgent())
resp renderMain(renderUserSearch(users, Prefs()), Prefs(), path=getPath())
of custom:
let tweets = await getSearch[Tweet](query, @"after", getAgent())
resp renderMain(renderTweetSearch(tweets, Prefs(), getPath()), Prefs(), path=getPath())
else:
resp Http404

View file

@ -3,14 +3,14 @@ import asyncdispatch, strutils, sequtils, uri
import jester import jester
import router_utils import router_utils
import ".."/[api, prefs, types, utils, cache, formatters, agents, search] import ".."/[api, prefs, types, utils, cache, formatters, agents, query]
import ../views/[general, profile, timeline, status] import ../views/[general, profile, timeline, status, search]
include "../views/rss.nimf" include "../views/rss.nimf"
export uri, sequtils export uri, sequtils
export router_utils export router_utils
export api, cache, formatters, search, agents export api, cache, formatters, query, agents
export profile, timeline, status export profile, timeline, status
type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto]) type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
@ -33,7 +33,7 @@ proc fetchSingleTimeline*(name, after, agent: string;
(profile, timeline) = await getProfileAndTimeline(name, agent, after) (profile, timeline) = await getProfileAndTimeline(name, agent, after)
cache(profile) cache(profile)
else: else:
var timelineFut = getTimelineSearch(get(query), after, agent) var timelineFut = getSearch[Tweet](get(query), after, agent)
if cachedProfile.isNone: if cachedProfile.isNone:
profile = await getCachedProfile(name, agent) profile = await getCachedProfile(name, agent)
timeline = await timelineFut timeline = await timelineFut
@ -49,7 +49,7 @@ proc fetchMultiTimeline*(names: seq[string]; after, agent: string;
else: else:
q = some(Query(kind: multi, fromUser: names, excludes: @["replies"])) q = some(Query(kind: multi, fromUser: names, excludes: @["replies"]))
return await getTimelineSearch(get(q), after, agent) return await getSearch[Tweet](get(q), after, agent)
proc showTimeline*(name, after: string; query: Option[Query]; proc showTimeline*(name, after: string; query: Option[Query];
prefs: Prefs; path, title, rss: string): Future[string] {.async.} = prefs: Prefs; path, title, rss: string): Future[string] {.async.} =

View file

@ -43,6 +43,14 @@
top: 50px; top: 50px;
} }
.profile-result .username {
margin: 0 !important;
}
.profile-result .tweet-header {
margin-bottom: unset;
}
@media(max-width: 600px) { @media(max-width: 600px) {
.profile-tabs { .profile-tabs {
width: 100vw; width: 100vw;

View file

@ -20,6 +20,14 @@
display: block; display: block;
font-weight: bold; font-weight: bold;
margin-bottom: 5px; margin-bottom: 5px;
input[type="text"] {
height: 20px;
}
button {
float: unset;
}
} }
.tab { .tab {

View file

@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video])
type type
QueryKind* = enum QueryKind* = enum
replies, media, multi, custom = "search" replies, media, multi, users, custom
Query* = object Query* = object
kind*: QueryKind kind*: QueryKind

View file

@ -60,13 +60,6 @@ proc renderMain*(body: VNode; prefs: Prefs; title="Nitter"; titleText=""; desc="
result = doctype & $node result = doctype & $node
proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")):
tdiv(class="search-panel"):
form(`method`="post", action="/search"):
input(`type`="text", name="query", autofocus="", placeholder="Enter usernames...")
button(`type`="submit"): icon "search"
proc renderError*(error: string): VNode = proc renderError*(error: string): VNode =
buildHtml(tdiv(class="panel-container")): buildHtml(tdiv(class="panel-container")):
tdiv(class="error-panel"): tdiv(class="error-panel"):

View file

@ -98,5 +98,5 @@ proc renderProfile*(profile: Profile; timeline: Timeline;
if profile.protected: if profile.protected:
renderProtected(profile.username) renderProtected(profile.username)
else: else:
renderProfileTabs(timeline, profile.username) renderProfileTabs(timeline.query, profile.username)
renderTimelineTweets(timeline, prefs, path) renderTimelineTweets(timeline, prefs, path)

38
src/views/search.nim Normal file
View file

@ -0,0 +1,38 @@
import strutils, strformat
import karax/[karaxdsl, vdom, vstyles]
import renderutils, timeline
import ".."/[types, formatters, query]
proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")):
tdiv(class="search-panel"):
form(`method`="get", action="/search"):
verbatim "<input name=\"kind\" style=\"display: none\" value=\"users\"/>"
input(`type`="text", name="text", autofocus="", placeholder="Enter username...")
button(`type`="submit"): icon "search"
proc renderTweetSearch*(timeline: Timeline; prefs: Prefs; path: string): VNode =
let users = get(timeline.query).fromUser
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
text users.join(" | ")
renderProfileTabs(timeline.query, users.join(","))
renderTimelineTweets(timeline, prefs, path)
proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode =
let searchText =
if users.query.isSome: get(users.query).text
else: ""
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
form(`method`="get", action="/search"):
verbatim "<input name=\"kind\" style=\"display: none\" value=\"users\"/>"
verbatim "<input type=\"text\" name=\"text\" value=\"$1\"/>" % searchText
button(`type`="submit"): icon "search"
renderSearchTabs(users.query)
renderTimelineUsers(users, prefs)

View file

@ -12,27 +12,35 @@ proc getQuery(query: Option[Query]): string =
if result[^1] != '?': if result[^1] != '?':
result &= "&" result &= "&"
proc getTabClass(results: Result; tab: string): string = proc getTabClass(query: Option[Query]; tab: string): string =
var classes = @["tab-item"] var classes = @["tab-item"]
if results.query.isNone or get(results.query).kind == multi: if query.isNone or get(query).kind == multi:
if tab == "posts": if tab == "posts":
classes.add "active" classes.add "active"
elif $get(results.query).kind == tab: elif $get(query).kind == tab:
classes.add "active" classes.add "active"
return classes.join(" ") return classes.join(" ")
proc renderProfileTabs*(timeline: Timeline; username: string): VNode = proc renderProfileTabs*(query: Option[Query]; username: string): VNode =
let link = "/" & username let link = "/" & username
buildHtml(ul(class="tab")): buildHtml(ul(class="tab")):
li(class=timeline.getTabClass("posts")): li(class=query.getTabClass("posts")):
a(href=link): text "Tweets" a(href=link): text "Tweets"
li(class=timeline.getTabClass("replies")): li(class=query.getTabClass("replies")):
a(href=(link & "/replies")): text "Tweets & Replies" a(href=(link & "/replies")): text "Tweets & Replies"
li(class=timeline.getTabClass("media")): li(class=query.getTabClass("media")):
a(href=(link & "/media")): text "Media" a(href=(link & "/media")): text "Media"
proc renderSearchTabs*(query: Option[Query]): VNode =
var q = if query.isSome: get(query) else: Query()
buildHtml(ul(class="tab")):
li(class=query.getTabClass("users")):
q.kind = users
a(href=genQueryUrl(q)): text "Users"
proc renderNewer(query: Option[Query]): VNode = proc renderNewer(query: Option[Query]): VNode =
buildHtml(tdiv(class="timeline-item show-more")): buildHtml(tdiv(class="timeline-item show-more")):
a(href=(getQuery(query).strip(chars={'?', '&'}))): a(href=(getQuery(query).strip(chars={'?', '&'}))):
@ -62,6 +70,34 @@ proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
proc threadFilter(it: Tweet; tweetThread: string): bool = proc threadFilter(it: Tweet; tweetThread: string): bool =
it.retweet.isNone and it.reply.len == 0 and it.threadId == tweetThread it.retweet.isNone and it.reply.len == 0 and it.threadId == tweetThread
proc renderUser(user: Profile; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-item")):
tdiv(class="tweet-body profile-result"):
tdiv(class="tweet-header"):
a(class="tweet-avatar", href=("/" & user.username)):
genImg(user.getUserpic("_bigger"), class="avatar")
tdiv(class="tweet-name-row"):
tdiv(class="fullname-and-username"):
linkUser(user, class="fullname")
linkUser(user, class="username")
tdiv(class="tweet-content media-body"):
verbatim linkifyText(user.bio, prefs)
proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline")):
if not results.beginning:
renderNewer(results.query)
if results.content.len > 0:
for user in results.content:
renderUser(user, prefs)
renderOlder(results.query, results.minId)
elif results.beginning:
renderNoneFound()
else:
renderNoMore()
proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string): VNode = proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string): VNode =
buildHtml(tdiv(class="timeline")): buildHtml(tdiv(class="timeline")):