Merge branch 'zedeus:master' into master

This commit is contained in:
PrivacyDevel 2023-07-21 22:38:31 +00:00 committed by GitHub
commit b2cc63cd99
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 55 additions and 17 deletions

View file

@ -33,6 +33,13 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
js = await fetch(url ? params, apiId) js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after) result = parseGraphTimeline(js, "user", after)
proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
if id.len == 0: return
let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = oldUserTweets / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return if id.len == 0: return
let let
@ -198,7 +205,7 @@ proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
ps = genParams({"screen_name": name, "trim_user": "true"}, ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false) count="18", ext=false)
url = photoRail ? ps url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.timeline)) result = parsePhotoRail(await fetch(url, Api.photoRail))
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} = proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0) let client = newAsyncHttpClient(maxRedirects=0)

View file

@ -14,6 +14,8 @@ const
userSearch* = api / "1.1/users/search.json" userSearch* = api / "1.1/users/search.json"
tweetSearch* = api / "1.1/search/tweets.json" tweetSearch* = api / "1.1/search/tweets.json"
oldUserTweets* = api / "2/timeline/profile"
graphql = api / "graphql" graphql = api / "graphql"
graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery" graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
@ -105,6 +107,18 @@ const
"includeHasBirdwatchNotes": false "includeHasBirdwatchNotes": false
}""" }"""
oldUserTweetsVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withV2Timeline": true
}
"""
userTweetsVariables* = """{ userTweetsVariables* = """{
"rest_id": "$1", $2 "rest_id": "$1", $2
"count": 20 "count": 20

View file

@ -83,7 +83,7 @@ routes:
error BadClientError: error BadClientError:
echo error.exc.name, ": ", error.exc.msg echo error.exc.name, ": ", error.exc.msg
resp Http500, showError("Network error occured, please try again.", cfg) resp Http500, showError("Network error occurred, please try again.", cfg)
error RateLimitError: error RateLimitError:
const link = a("another instance", href = instancesUrl) const link = a("another instance", href = instancesUrl)

View file

@ -336,6 +336,16 @@ proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
else: else:
result.retweet = some Tweet() result.retweet = some Tweet()
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return
let id = pin.getId
if id notin global.tweets: return
global.tweets[id].pinned = true
return finalizeTweet(global, id)
proc parseGlobalObjects(js: JsonNode): GlobalObjects = proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects() result = GlobalObjects()
let let
@ -346,24 +356,28 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result.users[k] = parseUser(v, k) result.users[k] = parseUser(v, k)
for k, v in tweets: for k, v in tweets:
var tweet = parseTweet(v, v{"tweet_card"}) var tweet = parseTweet(v, v{"card"})
if tweet.user.id in result.users: if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id] tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet result.tweets[k] = tweet
proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) = proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0: if js.kind != JArray or js.len == 0:
return return
for i in js: for i in js:
if res.tweets.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
res.pinned = some pin
with r, i{"replaceEntry", "entry"}: with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr: if "top" in r{"entryId"}.getStr:
res.top = r.getCursor res.tweets.top = r.getCursor
elif "bottom" in r{"entryId"}.getStr: elif "bottom" in r{"entryId"}.getStr:
res.bottom = r.getCursor res.tweets.bottom = r.getCursor
proc parseTimeline*(js: JsonNode; after=""): Timeline = proc parseTimeline*(js: JsonNode; after=""): Profile =
result = Timeline(beginning: after.len == 0) result = Profile(tweets: Timeline(beginning: after.len == 0))
let global = parseGlobalObjects(? js) let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"} let instructions = ? js{"timeline", "instructions"}
@ -381,17 +395,17 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry: if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId) let tweet = finalizeTweet(global, e.getEntryId)
if not tweet.available: continue if not tweet.available: continue
result.content.add tweet result.tweets.content.add tweet
elif "cursor-top" in entry: elif "cursor-top" in entry:
result.top = e.getCursor result.tweets.top = e.getCursor
elif "cursor-bottom" in entry: elif "cursor-bottom" in entry:
result.bottom = e.getCursor result.tweets.bottom = e.getCursor
elif entry.startsWith("sq-cursor"): elif entry.startsWith("sq-cursor"):
with cursor, e{"content", "operation", "cursor"}: with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom": if cursor{"cursorType"}.getStr == "Bottom":
result.bottom = cursor{"value"}.getStr result.tweets.bottom = cursor{"value"}.getStr
else: else:
result.top = cursor{"value"}.getStr result.tweets.top = cursor{"value"}.getStr
proc parsePhotoRail*(js: JsonNode): PhotoRail = proc parsePhotoRail*(js: JsonNode): PhotoRail =
with error, js{"error"}: with error, js{"error"}:

View file

@ -54,7 +54,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
result = result =
case query.kind case query.kind
of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) of posts: await getTimeline(userId, after)
of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
of media: await getGraphUserTweets(userId, TimelineKind.media, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after)
of favorites: Profile(tweets: await getFavorites(userId, cfg, after)) of favorites: Profile(tweets: await getFavorites(userId, cfg, after))

View file

@ -41,8 +41,10 @@ proc getPoolJson*(): JsonNode =
let let
maxReqs = maxReqs =
case api case api
of Api.timeline, Api.search: 180 of Api.photoRail, Api.search: 180
of Api.userTweets, Api.userTweetsAndReplies, Api.userRestId, of Api.timeline: 187
of Api.userTweets: 300
of Api.userTweetsAndReplies, Api.userRestId,
Api.userScreenName, Api.tweetDetail, Api.tweetResult: 500 Api.userScreenName, Api.tweetDetail, Api.tweetResult: 500
of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia, Api.favorites, Api.retweeters, Api.favoriters: 500 of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia, Api.favorites, Api.retweeters, Api.favoriters: 500
of Api.userSearch: 900 of Api.userSearch: 900

View file

@ -18,6 +18,7 @@ type
tweetDetail tweetDetail
tweetResult tweetResult
timeline timeline
photoRail
search search
userSearch userSearch
list list

View file

@ -11,7 +11,7 @@ const doctype = "<!DOCTYPE html>\n"
proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string = proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
let thumb = get(tweet.video).thumb let thumb = get(tweet.video).thumb
let vidUrl = getVideoEmbed(cfg, tweet.id) let vidUrl = getVideoEmbed(cfg, tweet.id)
let prefs = Prefs(hlsPlayback: true) let prefs = Prefs(hlsPlayback: true, mp4Playback: true)
let node = buildHtml(html(lang="en")): let node = buildHtml(html(lang="en")):
renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb])) renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))