Switch to TweetDetail for tweets

This commit is contained in:
Zed 2023-08-25 16:28:30 +02:00
parent 88b005c9da
commit ae9fa02bf5
6 changed files with 31 additions and 21 deletions

View file

@ -17,7 +17,7 @@ const
graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2" graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2"
graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2" graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2"
graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2" graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
graphTweet* = graphql / "83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2" graphTweet* = graphql / "q94uRCEn65LZThakYcPT6g/TweetDetail"
graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery" graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline" graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId" graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
@ -89,8 +89,12 @@ const
tweetVariables* = """{ tweetVariables* = """{
"focalTweetId": "$1", "focalTweetId": "$1",
$2 $2
"includeHasBirdwatchNotes": false "includeHasBirdwatchNotes": false,
}""" "includePromotedContent": false,
"withBirdwatchNotes": false,
"withVoice": false,
"withV2Timeline": true
}""".replace(" ", "").replace("\n", "")
# oldUserTweetsVariables* = """{ # oldUserTweetsVariables* = """{
# "userId": "$1", $2 # "userId": "$1", $2

View file

@ -324,7 +324,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
of "TweetWithVisibilityResults": of "TweetWithVisibilityResults":
return parseGraphTweet(js{"tweet"}) return parseGraphTweet(js{"tweet"})
var jsCard = copy(js{"tweet_card", "legacy"}) var jsCard = copy(js{"card", "legacy"})
if jsCard.kind != JNull: if jsCard.kind != JNull:
var values = newJObject() var values = newJObject()
for val in jsCard["binding_values"]: for val in jsCard["binding_values"]:
@ -342,7 +342,6 @@ proc parseGraphTweet(js: JsonNode): Tweet =
result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"})) result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
let thread = js{"content", "items"}
for t in js{"content", "items"}: for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr let entryId = t{"entryId"}.getStr
if "cursor-showmore" in entryId: if "cursor-showmore" in entryId:
@ -350,11 +349,16 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
result.thread.cursor = cursor.getStr result.thread.cursor = cursor.getStr
result.thread.hasMore = true result.thread.hasMore = true
elif "tweet" in entryId: elif "tweet" in entryId:
let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"}) let
result.thread.content.add tweet isLegacy = t{"item"}.hasKey("itemContent")
(contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
else: ("content", "tweetResult")
if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread": with content, t{"item", contentKey}:
result.self = true result.thread.content.add parseGraphTweet(content{resultKey, "result"})
if content{"tweetDisplayType"}.getStr == "SelfThread":
result.self = true
proc parseGraphTweetResult*(js: JsonNode): Tweet = proc parseGraphTweetResult*(js: JsonNode): Tweet =
with tweet, js{"data", "tweet_result", "result"}: with tweet, js{"data", "tweet_result", "result"}:
@ -363,14 +367,14 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet =
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true)) result = Conversation(replies: Result[Chain](beginning: true))
let instructions = ? js{"data", "timeline_response", "instructions"} let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
if instructions.len == 0: if instructions.len == 0:
return return
for e in instructions[0]{"entries"}: for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"): if entryId.startsWith("tweet"):
with tweetResult, e{"content", "content", "tweetResult", "result"}: with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult) let tweet = parseGraphTweet(tweetResult)
if not tweet.available: if not tweet.available:
@ -385,7 +389,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
let tweet = Tweet( let tweet = Tweet(
id: parseBiggestInt(id), id: parseBiggestInt(id),
available: false, available: false,
text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
) )
if id == tweetId: if id == tweetId:
@ -397,9 +401,10 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
if self: if self:
result.after = thread result.after = thread
else: else:
echo "adding thread: ", thread.content.len
result.replies.content.add thread result.replies.content.add thread
elif entryId.startsWith("cursor-bottom"): elif entryId.startsWith("cursor-bottom"):
result.replies.bottom = e{"content", "content", "value"}.getStr result.replies.bottom = e{"content", "itemContent", "value"}.getStr
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0)) result = Profile(tweets: Timeline(beginning: after.len == 0))

View file

@ -52,10 +52,11 @@ proc getPoolJson*(): JsonNode =
maxReqs = maxReqs =
case api case api
of Api.search: 50 of Api.search: 50
of Api.tweetDetail: 150
of Api.photoRail: 180 of Api.photoRail: 180
of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
Api.userRestId, Api.userScreenName, Api.userRestId, Api.userScreenName,
Api.tweetResult, Api.tweetDetail, Api.tweetResult,
Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500 Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
of Api.userSearch: 900 of Api.userSearch: 900
reqs = maxReqs - apiStatus.remaining reqs = maxReqs - apiStatus.remaining

View file

@ -13,11 +13,6 @@ card = [
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim', 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
'gist.github.com', True], 'gist.github.com', True],
['FluentAI/status/1116417904831029248',
'Amazons Alexa isnt just AI — thousands of humans are listening',
'One of the only ways to improve Alexa is to have human beings check it for errors',
'theverge.com', True],
['nim_lang/status/1082989146040340480', ['nim_lang/status/1082989146040340480',
'Nim in 2018: A short recap', 'Nim in 2018: A short recap',
'There were several big news in the Nim world in 2018 two new major releases, partnership with Status, and much more. But let us go chronologically.', 'There were several big news in the Nim world in 2018 two new major releases, partnership with Status, and much more. But let us go chronologically.',
@ -25,6 +20,11 @@ card = [
] ]
no_thumb = [ no_thumb = [
['FluentAI/status/1116417904831029248',
'Amazons Alexa isnt just AI — thousands of humans are listening',
'One of the only ways to improve Alexa is to have human beings check it for errors',
'theverge.com'],
['Thom_Wolf/status/1122466524860702729', ['Thom_Wolf/status/1122466524860702729',
'facebookresearch/fairseq', 'facebookresearch/fairseq',
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.', 'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',

View file

@ -6,7 +6,7 @@ normal = [['jack'], ['elonmusk']]
after = [['jack', '1681686036294803456'], after = [['jack', '1681686036294803456'],
['elonmusk', '1681686036294803456']] ['elonmusk', '1681686036294803456']]
no_more = [['mobile_test_8?cursor=1000']] no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
empty = [['emptyuser'], ['mobile_test_10']] empty = [['emptyuser'], ['mobile_test_10']]

View file

@ -14,7 +14,7 @@ poll = [
image = [ image = [
['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'], ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj'] #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
] ]
gif = [ gif = [