Merge remote-tracking branch 'upstream/master'

This commit is contained in:
PrivacyDev 2023-07-22 09:30:11 -04:00
commit b2beabf6cd
8 changed files with 180 additions and 166 deletions

View file

@ -33,12 +33,12 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after)
proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
if id.len == 0: return
let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = oldUserTweets / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
# proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
# if id.len == 0: return
# let
# ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
# url = oldUserTweets / (id & ".json") ? ps
# result = parseTimeline(await fetch(url, Api.timeline), after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
@ -166,20 +166,22 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
result.tweets.query = query
proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
let q = genQueryParam(query)
var q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Timeline(query: query, beginning: true)
if after.len > 0:
q &= " max_id:" & after
let url = tweetSearch ? genParams({
"q": q,
"tweet_search_mode": "live",
"max_id": after
"q": q ,
"modules": "status",
"result_type": "recent",
})
result = parseTweetSearch(await fetch(url, Api.search))
result = parseTweetSearch(await fetch(url, Api.search), after)
result.query = query
if after.len == 0:
result.beginning = true
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
if query.text.len == 0:

View file

@ -17,7 +17,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
for p in pars:
result &= p
if ext:
result &= ("ext", "mediaStats")
result &= ("ext", "mediaStats,isBlueVerified,isVerified,blue,blueVerified")
result &= ("include_ext_alt_text", "1")
result &= ("include_ext_media_availability", "1")
if count.len > 0:

View file

@ -12,9 +12,9 @@ const
timelineApi = api / "2/timeline"
favorites* = timelineApi / "favorites"
userSearch* = api / "1.1/users/search.json"
tweetSearch* = api / "1.1/search/tweets.json"
tweetSearch* = api / "1.1/search/universal.json"
oldUserTweets* = api / "2/timeline/profile"
# oldUserTweets* = api / "2/timeline/profile"
graphql = api / "graphql"
graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
@ -35,27 +35,28 @@ const
graphFollowing* = graphql / "JPZiqKjET7_M1r5Tlr8pyA/Following"
timelineParams* = {
"include_profile_interstitial_type": "0",
"include_blocking": "0",
"include_blocked_by": "0",
"include_followed_by": "0",
"include_want_retweets": "0",
"include_mute_edge": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_ext_is_blue_verified": "1",
"skip_status": "1",
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "0",
"include_reply_count": "1",
"cards_platform": "Web-13",
"tweet_mode": "extended",
"include_entities": "1",
"include_user_entities": "1",
"include_ext_media_color": "0",
"ui_lang": "en-US",
"send_error_codes": "1",
"simple_quoted_tweet": "1",
"include_quote_count": "1"
"skip_status": "1",
"include_blocked_by": "0",
"include_blocking": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_cards": "1",
"include_composer_source": "0",
"include_entities": "1",
"include_ext_is_blue_verified": "1",
"include_ext_media_color": "0",
"include_followed_by": "0",
"include_mute_edge": "0",
"include_profile_interstitial_type": "0",
"include_quote_count": "1",
"include_reply_count": "1",
"include_user_entities": "1",
"include_want_retweets": "0",
}.toSeq
gqlFeatures* = """{
@ -107,17 +108,17 @@ const
"includeHasBirdwatchNotes": false
}"""
oldUserTweetsVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withV2Timeline": true
}
"""
# oldUserTweetsVariables* = """{
# "userId": "$1", $2
# "count": 20,
# "includePromotedContent": false,
# "withDownvotePerspective": false,
# "withReactionsMetadata": false,
# "withReactionsPerspective": false,
# "withVoice": false,
# "withV2Timeline": true
# }
# """
userTweetsVariables* = """{
"rest_id": "$1", $2

View file

@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, options, tables, times, math
import strutils, options, times, math
import packedjson, packedjson/deserialiser
import types, parserutils, utils
import experimental/parser/unifiedcard
@ -302,110 +302,112 @@ proc parseLegacyTweet(js: JsonNode): Tweet =
if result.quote.isSome:
result.quote = some parseLegacyTweet(js{"quoted_status"})
proc parseTweetSearch*(js: JsonNode): Timeline =
if js.kind == JNull or "statuses" notin js:
return Timeline(beginning: true)
proc parseTweetSearch*(js: JsonNode; after=""): Timeline =
result.beginning = after.len == 0
for tweet in js{"statuses"}:
let parsed = parseLegacyTweet(tweet)
if parsed.retweet.isSome:
parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})
result.content.add @[parsed]
let cursor = js{"search_metadata", "next_results"}.getStr
if cursor.len > 0 and "max_id" in cursor:
result.bottom = cursor[cursor.find("=") + 1 .. cursor.find("&q=")]
proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
let intId = if id.len > 0: parseBiggestInt(id) else: 0
result = global.tweets.getOrDefault(id, Tweet(id: intId))
if result.quote.isSome:
let quote = get(result.quote).id
if $quote in global.tweets:
result.quote = some global.tweets[$quote]
else:
result.quote = some Tweet()
if result.retweet.isSome:
let rt = get(result.retweet).id
if $rt in global.tweets:
result.retweet = some finalizeTweet(global, $rt)
else:
result.retweet = some Tweet()
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return
let id = pin.getId
if id notin global.tweets: return
global.tweets[id].pinned = true
return finalizeTweet(global, id)
proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects()
let
tweets = ? js{"globalObjects", "tweets"}
users = ? js{"globalObjects", "users"}
for k, v in users:
result.users[k] = parseUser(v, k)
for k, v in tweets:
var tweet = parseTweet(v, v{"card"})
if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet
proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0:
if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0:
return
for i in js:
if res.tweets.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
res.pinned = some pin
for item in js{"modules"}:
with tweet, item{"status", "data"}:
let parsed = parseLegacyTweet(tweet)
with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr:
res.tweets.top = r.getCursor
elif "bottom" in r{"entryId"}.getStr:
res.tweets.bottom = r.getCursor
if parsed.retweet.isSome:
parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})
proc parseTimeline*(js: JsonNode; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0))
let global = parseGlobalObjects(? js)
result.content.add @[parsed]
let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0: return
if result.content.len > 0:
result.bottom = $(result.content[^1][0].id - 1)
result.parseInstructions(global, instructions)
# proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
# let intId = if id.len > 0: parseBiggestInt(id) else: 0
# result = global.tweets.getOrDefault(id, Tweet(id: intId))
var entries: JsonNode
for i in instructions:
if "addEntries" in i:
entries = i{"addEntries", "entries"}
# if result.quote.isSome:
# let quote = get(result.quote).id
# if $quote in global.tweets:
# result.quote = some global.tweets[$quote]
# else:
# result.quote = some Tweet()
for e in ? entries:
let entry = e{"entryId"}.getStr
if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
if not tweet.available: continue
result.tweets.content.add tweet
elif "cursor-top" in entry:
result.tweets.top = e.getCursor
elif "cursor-bottom" in entry:
result.tweets.bottom = e.getCursor
elif entry.startsWith("sq-cursor"):
with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom":
result.tweets.bottom = cursor{"value"}.getStr
else:
result.tweets.top = cursor{"value"}.getStr
# if result.retweet.isSome:
# let rt = get(result.retweet).id
# if $rt in global.tweets:
# result.retweet = some finalizeTweet(global, $rt)
# else:
# result.retweet = some Tweet()
# proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
# let pin = js{"pinEntry", "entry", "entryId"}.getStr
# if pin.len == 0: return
# let id = pin.getId
# if id notin global.tweets: return
# global.tweets[id].pinned = true
# return finalizeTweet(global, id)
# proc parseGlobalObjects(js: JsonNode): GlobalObjects =
# result = GlobalObjects()
# let
# tweets = ? js{"globalObjects", "tweets"}
# users = ? js{"globalObjects", "users"}
# for k, v in users:
# result.users[k] = parseUser(v, k)
# for k, v in tweets:
# var tweet = parseTweet(v, v{"card"})
# if tweet.user.id in result.users:
# tweet.user = result.users[tweet.user.id]
# result.tweets[k] = tweet
# proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
# if js.kind != JArray or js.len == 0:
# return
# for i in js:
# if res.tweets.beginning and i{"pinEntry"}.notNull:
# with pin, parsePin(i, global):
# res.pinned = some pin
# with r, i{"replaceEntry", "entry"}:
# if "top" in r{"entryId"}.getStr:
# res.tweets.top = r.getCursor
# elif "bottom" in r{"entryId"}.getStr:
# res.tweets.bottom = r.getCursor
# proc parseTimeline*(js: JsonNode; after=""): Profile =
# result = Profile(tweets: Timeline(beginning: after.len == 0))
# let global = parseGlobalObjects(? js)
# let instructions = ? js{"timeline", "instructions"}
# if instructions.len == 0: return
# result.parseInstructions(global, instructions)
# var entries: JsonNode
# for i in instructions:
# if "addEntries" in i:
# entries = i{"addEntries", "entries"}
# for e in ? entries:
# let entry = e{"entryId"}.getStr
# if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
# let tweet = finalizeTweet(global, e.getEntryId)
# if not tweet.available: continue
# result.tweets.content.add tweet
# elif "cursor-top" in entry:
# result.tweets.top = e.getCursor
# elif "cursor-bottom" in entry:
# result.tweets.bottom = e.getCursor
# elif entry.startsWith("sq-cursor"):
# with cursor, e{"content", "operation", "cursor"}:
# if cursor{"cursorType"}.getStr == "Bottom":
# result.tweets.bottom = cursor{"value"}.getStr
# else:
# result.tweets.top = cursor{"value"}.getStr
proc parsePhotoRail*(js: JsonNode): PhotoRail =
with error, js{"error"}:

View file

@ -54,7 +54,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
result =
case query.kind
of posts: await getTimeline(userId, after)
# of posts: await getTimeline(userId, after)
of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
of media: await getGraphUserTweets(userId, TimelineKind.media, after)
of favorites: await getFavorites(userId, cfg, after)
@ -63,10 +63,18 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
result.user = await user
result.photoRail = await rail
result.tweets.query = query
if result.user.protected or result.user.suspended:
return
result.tweets.query = query
if not skipPinned and query.kind == posts and
result.user.pinnedTweet > 0 and after.len == 0:
let tweet = await getCachedTweet(result.user.pinnedTweet)
if not tweet.isNil:
tweet.pinned = true
tweet.user = result.user
result.pinned = some tweet
proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
rss, after: string): Future[string] {.async.} =

View file

@ -41,7 +41,8 @@ proc getPoolJson*(): JsonNode =
let
maxReqs =
case api
of Api.photoRail, Api.search: 180
of Api.search: 100000
of Api.photoRail: 180
of Api.timeline: 187
of Api.userTweets: 300
of Api.userTweetsAndReplies, Api.userRestId,

View file

@ -1,12 +1,12 @@
from base import BaseTestCase, Timeline
from parameterized import parameterized
normal = [['mobile_test'], ['mobile_test_2']]
normal = [['jack'], ['elonmusk']]
after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'],
['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']]
after = [['jack', '1681686036294803456'],
['elonmusk', '1681686036294803456']]
no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']]
no_more = [['mobile_test_8?cursor=1000']]
empty = [['emptyuser'], ['mobile_test_10']]

View file

@ -80,16 +80,16 @@ retweet = [
class TweetTest(BaseTestCase):
@parameterized.expand(timeline)
def test_timeline(self, index, fullname, username, date, tid, text):
self.open_nitter(username)
tweet = get_timeline_tweet(index)
self.assert_exact_text(fullname, tweet.fullname)
self.assert_exact_text('@' + username, tweet.username)
self.assert_exact_text(date, tweet.date)
self.assert_text(text, tweet.text)
permalink = self.find_element(tweet.date + ' a')
self.assertIn(tid, permalink.get_attribute('href'))
# @parameterized.expand(timeline)
# def test_timeline(self, index, fullname, username, date, tid, text):
# self.open_nitter(username)
# tweet = get_timeline_tweet(index)
# self.assert_exact_text(fullname, tweet.fullname)
# self.assert_exact_text('@' + username, tweet.username)
# self.assert_exact_text(date, tweet.date)
# self.assert_text(text, tweet.text)
# permalink = self.find_element(tweet.date + ' a')
# self.assertIn(tid, permalink.get_attribute('href'))
@parameterized.expand(status)
def test_status(self, tid, fullname, username, date, text):
@ -123,14 +123,14 @@ class TweetTest(BaseTestCase):
link = self.find_link_text(f'@{un}')
self.assertIn(f'/{un}', link.get_property('href'))
@parameterized.expand(retweet)
def test_retweet(self, index, url, retweet_by, fullname, username, text):
self.open_nitter(url)
tweet = get_timeline_tweet(index)
self.assert_text(f'{retweet_by} retweeted', tweet.retweet)
self.assert_text(text, tweet.text)
self.assert_exact_text(fullname, tweet.fullname)
self.assert_exact_text(username, tweet.username)
# @parameterized.expand(retweet)
# def test_retweet(self, index, url, retweet_by, fullname, username, text):
# self.open_nitter(url)
# tweet = get_timeline_tweet(index)
# self.assert_text(f'{retweet_by} retweeted', tweet.retweet)
# self.assert_text(text, tweet.text)
# self.assert_exact_text(fullname, tweet.fullname)
# self.assert_exact_text(username, tweet.username)
@parameterized.expand(invalid)
def test_invalid_id(self, tweet):