diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 37979cb..ee28e33 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -10,25 +10,34 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: buildjet-2vcpu-ubuntu-2204 + strategy: + matrix: + nim: + - "1.6.10" + - "1.6.x" + - "2.0.x" + - "devel" steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache nimble id: cache-nimble - uses: actions/cache@v3 + uses: buildjet/cache@v3 with: path: ~/.nimble - key: nimble-${{ hashFiles('*.nimble') }} - restore-keys: "nimble-" + key: ${{ matrix.nim }}-nimble-${{ hashFiles('*.nimble') }} + restore-keys: | + ${{ matrix.nim }}-nimble- - uses: actions/setup-python@v4 with: python-version: "3.10" cache: "pip" - uses: jiro4989/setup-nim-action@v1 with: - nim-version: "1.x" + nim-version: ${{ matrix.nim }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - run: nimble build -d:release -Y - run: pip install seleniumbase - run: seleniumbase install chromedriver @@ -37,12 +46,11 @@ jobs: run: | sudo apt install libsass-dev -y cp nitter.example.conf nitter.conf + sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf nimble md nimble scss + echo '${{ secrets.GUEST_ACCOUNTS }}' > ./guest_accounts.jsonl - name: Run tests - env: - GUEST_ACCOUNTS: ${{ secrets.GUEST_ACCOUNTS }} run: | - echo $GUEST_ACCOUNTS > ./guest_accounts.json ./nitter & - pytest -n4 tests + pytest -n8 tests diff --git a/.gitignore b/.gitignore index d43cc3f..ea520dc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ nitter /public/css/style.css /public/md/*.html nitter.conf +guest_accounts.json* dump.rdb diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 6cd6744..70024b2 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1,7 +1,7 @@ -FROM alpine:3.17 as nim +FROM alpine:3.18 as nim LABEL maintainer="setenforce@protonmail.com" -RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre +RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.14-r0" "nimble=0.13.1-r2" WORKDIR /src/nitter @@ -13,11 +13,13 @@ RUN nimble build -d:danger -d:lto -d:strip \ && nimble scss \ && nimble md -FROM alpine:3.17 +FROM alpine:3.18 WORKDIR /src/ -RUN apk --no-cache add ca-certificates pcre openssl1.1-compat +RUN apk --no-cache add pcre ca-certificates openssl1.1-compat COPY --from=nim /src/nitter/nitter ./ COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf COPY --from=nim /src/nitter/public ./public EXPOSE 8080 +RUN adduser -h /src/ -D -s /bin/sh nitter +USER nitter CMD ./nitter diff --git a/config.nims b/config.nims index b7e52d0..4a7af27 100644 --- a/config.nims +++ b/config.nims @@ -7,12 +7,7 @@ # disable annoying warnings warning("GcUnsafe2", off) +warning("HoleEnumConv", off) hint("XDeclaredButNotUsed", off) hint("XCannotRaiseY", off) hint("User", off) - -const - nimVersion = (major: NimMajor, minor: NimMinor, patch: NimPatch) - -when nimVersion >= (1, 6, 0): - warning("HoleEnumConv", off) diff --git a/nitter.example.conf b/nitter.example.conf index 8d078f1..ffe7a3b 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -23,7 +23,7 @@ redisMaxConnections = 30 hmacKey = "secretkey" # random key for cryptographic signing of video urls base64Media = false # use base64 encoding for proxied media urls enableRSS = true # set this to false to disable RSS feeds -enableDebug = false # enable request logs and debug endpoints (/.tokens) +enableDebug = false # enable request logs and debug endpoints (/.accounts) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" tokenCount = 10 diff --git a/nitter.nimble b/nitter.nimble index e6a1909..37f9229 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -10,11 +10,11 @@ bin = @["nitter"] # Dependencies -requires "nim >= 1.4.8" +requires "nim >= 1.6.10" requires "jester#baca3f" requires "karax#5cf360c" requires "sass#7dfdd03" -requires "nimcrypto#4014ef9" +requires "nimcrypto#a079df9" requires "markdown#158efe3" requires "packedjson#9e6fbb6" requires "supersnappy#6c94198" @@ -22,7 +22,7 @@ requires "redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#ca5989a" requires "flatty#e668085" -requires "jsony#ea811be" +requires "jsony#1de1f08" requires "oauth#b8c163b" # Tasks diff --git a/src/api.nim b/src/api.nim index 3e470bc..d0bfd8c 100644 --- a/src/api.nim +++ b/src/api.nim @@ -155,25 +155,29 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = if after.len > 0: variables["cursor"] = % after let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphSearch(await fetch(url, Api.search), after) + result = parseGraphSearch[Tweets](await fetch(url, Api.search), after) result.query = query -proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = +proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} = if query.text.len == 0: return Result[User](query: query, beginning: true) - let - page = if page.len == 0: "1" else: page - url = userSearch ? genParams({"q": query.text, "skip_status": "1", "page": page}) - js = await fetchRaw(url, Api.userSearch) - - result = parseUsers(js) + var + variables = %*{ + "rawQuery": query.text, + "count": 20, + "product": "People", + "withDownvotePerspective": false, + "withReactionsMetadata": false, + "withReactionsPerspective": false + } + if after.len > 0: + variables["cursor"] = % after + result.beginning = false + let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} + result = parseGraphSearch[User](await fetch(url, Api.search), after) result.query = query - if page.len == 0: - result.bottom = "2" - elif page.allCharsInSet(Digits): - result.bottom = $(parseInt(page) + 1) proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = if name.len == 0: return diff --git a/src/apiutils.nim b/src/apiutils.nim index ab483dc..22e75c9 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import jsony, packedjson, zippy, oauth1 -import types, tokens, consts, parserutils, http_pool +import types, auth, consts, parserutils, http_pool import experimental/types/common import config @@ -134,7 +134,7 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = except OSError as e: raise e except Exception as e: - let id = if account.isNil: "null" else: account.id + let id = if account.isNil: "null" else: $account.id echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url raise rateLimitError() finally: diff --git a/src/tokens.nim b/src/auth.nim similarity index 52% rename from src/tokens.nim rename to src/auth.nim index fd5a14f..b288c50 100644 --- a/src/tokens.nim +++ b/src/auth.nim @@ -1,11 +1,27 @@ #SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, times, json, random, strutils, tables, sets +import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] import types +import experimental/parser/guestaccount # max requests at a time per account to avoid race conditions const maxConcurrentReqs = 2 dayInSeconds = 24 * 60 * 60 + apiMaxReqs: Table[Api, int] = { + Api.search: 50, + Api.tweetDetail: 150, + Api.photoRail: 180, + Api.userTweets: 500, + Api.userTweetsAndReplies: 500, + Api.userMedia: 500, + Api.userRestId: 500, + Api.userScreenName: 500, + Api.tweetResult: 500, + Api.list: 500, + Api.listTweets: 500, + Api.listMembers: 500, + Api.listBySlug: 500 + }.toTable var accountPool: seq[GuestAccount] @@ -14,20 +30,75 @@ var template log(str: varargs[string, `$`]) = if enableLogging: echo "[accounts] ", str.join("") -proc getPoolJson*(): JsonNode = - var - list = newJObject() - totalReqs = 0 - totalPending = 0 - limited: HashSet[string] - reqsPerApi: Table[string, int] +proc snowflakeToEpoch(flake: int64): int64 = + int64(((flake shr 22) + 1288834974657) div 1000) +proc hasExpired(account: GuestAccount): bool = + let + created = snowflakeToEpoch(account.id) + now = epochTime().int64 + daysOld = int(now - created) div dayInSeconds + return daysOld > 30 + +proc getAccountPoolHealth*(): JsonNode = let now = epochTime().int - for account in accountPool: - totalPending.inc(account.pending) + var + totalReqs = 0 + limited: PackedSet[int64] + reqsPerApi: Table[string, int] + oldest = now.int64 + newest = 0'i64 + average = 0'i64 - var includeAccount = false + for account in accountPool: + let created = snowflakeToEpoch(account.id) + if created > newest: + newest = created + if created < oldest: + oldest = created + average += created + + for api in account.apis.keys: + let + apiStatus = account.apis[api] + reqs = apiMaxReqs[api] - apiStatus.remaining + + if apiStatus.limited: + limited.incl account.id + + # no requests made with this account and endpoint since the limit reset + if apiStatus.reset < now: + continue + + reqsPerApi.mgetOrPut($api, 0).inc reqs + totalReqs.inc reqs + + if accountPool.len > 0: + average = average div accountPool.len + else: + oldest = 0 + average = 0 + + return %*{ + "accounts": %*{ + "total": accountPool.len, + "limited": limited.card, + "oldest": $fromUnix(oldest), + "newest": $fromUnix(newest), + "average": $fromUnix(average) + }, + "requests": %*{ + "total": totalReqs, + "apis": reqsPerApi + } + } + +proc getAccountPoolDebug*(): JsonNode = + let now = epochTime().int + var list = newJObject() + + for account in accountPool: let accountJson = %*{ "apis": newJObject(), "pending": account.pending, @@ -46,38 +117,11 @@ proc getPoolJson*(): JsonNode = if apiStatus.limited: obj["limited"] = %true - limited.incl account.id accountJson{"apis", $api} = obj - includeAccount = true + list[$account.id] = accountJson - let - maxReqs = - case api - of Api.search: 50 - of Api.tweetDetail: 150 - of Api.photoRail: 180 - of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, - Api.userRestId, Api.userScreenName, - Api.tweetResult, - Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.favorites, Api.retweeters, Api.favoriters, Api.following, Api.followers: 500 - of Api.userSearch: 900 - reqs = maxReqs - apiStatus.remaining - - reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs - totalReqs.inc(reqs) - - if includeAccount: - list[account.id] = accountJson - - return %*{ - "amount": accountPool.len, - "limited": limited.card, - "requests": totalReqs, - "pending": totalPending, - "apis": reqsPerApi, - "accounts": list - } + return %list proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") @@ -141,12 +185,25 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = account.apis[api] = RateLimit(remaining: remaining, reset: reset) -proc initAccountPool*(cfg: Config; accounts: JsonNode) = +proc initAccountPool*(cfg: Config; path: string) = enableLogging = cfg.enableDebug - for account in accounts: - accountPool.add GuestAccount( - id: account{"user", "id_str"}.getStr, - oauthToken: account{"oauth_token"}.getStr, - oauthSecret: account{"oauth_token_secret"}.getStr, - ) + let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path + + if fileExists(jsonlPath): + log "Parsing JSONL guest accounts file: ", jsonlPath + for line in jsonlPath.lines: + accountPool.add parseGuestAccount(line) + elif fileExists(path): + log "Parsing JSON guest accounts file: ", path + accountPool = parseGuestAccounts(path) + else: + echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." + quit 1 + + let accountsPrePurge = accountPool.len + accountPool.keepItIf(not it.hasExpired) + + log "Successfully added ", accountPool.len, " valid accounts." + if accountsPrePurge > accountPool.len: + log "Purged ", accountsPrePurge - accountPool.len, " expired accounts." diff --git a/src/consts.nim b/src/consts.nim index aef51c4..7d79c06 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -9,7 +9,6 @@ const activate* = $(api / "1.1/guest/activate.json") photoRail* = api / "1.1/statuses/media_timeline.json" - userSearch* = api / "1.1/users/search.json" timelineApi = api / "2/timeline" favorites* = timelineApi / "favorites" @@ -37,12 +36,10 @@ const "include_cards": "1", "include_entities": "1", "include_profile_interstitial_type": "0", - "include_quote_count": "1", - "include_reply_count": "1", - "include_user_entities": "1", - "include_ext_reply_count": "1", - "include_ext_is_blue_verified": "1", - #"include_ext_verified_type": "1", + "include_quote_count": "0", + "include_reply_count": "0", + "include_user_entities": "0", + "include_ext_reply_count": "0", "include_ext_media_color": "0", "cards_platform": "Web-13", "tweet_mode": "extended", diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index b9da7c4..69837ab 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -1,7 +1,7 @@ import options import jsony import user, ../types/[graphuser, graphlistmembers] -from ../../types import User, Result, Query, QueryKind +from ../../types import User, VerifiedType, Result, Query, QueryKind proc parseGraphUser*(json: string): User = if json.len == 0 or json[0] != '{': @@ -12,9 +12,10 @@ proc parseGraphUser*(json: string): User = if raw.data.userResult.result.unavailableReason.get("") == "Suspended": return User(suspended: true) - result = toUser raw.data.userResult.result.legacy + result = raw.data.userResult.result.legacy result.id = raw.data.userResult.result.restId - result.verified = result.verified or raw.data.userResult.result.isBlueVerified + if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified: + result.verifiedType = blue proc parseGraphListMembers*(json, cursor: string): Result[User] = result = Result[User]( @@ -30,7 +31,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] = of TimelineTimelineItem: let userResult = entry.content.itemContent.userResults.result if userResult.restId.len > 0: - result.content.add toUser userResult.legacy + result.content.add userResult.legacy of TimelineTimelineCursor: if entry.content.cursorType == "Bottom": result.bottom = entry.content.value diff --git a/src/experimental/parser/guestaccount.nim b/src/experimental/parser/guestaccount.nim new file mode 100644 index 0000000..f7e6d34 --- /dev/null +++ b/src/experimental/parser/guestaccount.nim @@ -0,0 +1,21 @@ +import std/strutils +import jsony +import ../types/guestaccount +from ../../types import GuestAccount + +proc toGuestAccount(account: RawAccount): GuestAccount = + let id = account.oauthToken[0 ..< account.oauthToken.find('-')] + result = GuestAccount( + id: parseBiggestInt(id), + oauthToken: account.oauthToken, + oauthSecret: account.oauthTokenSecret + ) + +proc parseGuestAccount*(raw: string): GuestAccount = + let rawAccount = raw.fromJson(RawAccount) + result = rawAccount.toGuestAccount + +proc parseGuestAccounts*(path: string): seq[GuestAccount] = + let rawAccounts = readFile(path).fromJson(seq[RawAccount]) + for account in rawAccounts: + result.add account.toGuestAccount diff --git a/src/experimental/parser/unifiedcard.nim b/src/experimental/parser/unifiedcard.nim index c9af437..a112974 100644 --- a/src/experimental/parser/unifiedcard.nim +++ b/src/experimental/parser/unifiedcard.nim @@ -1,6 +1,6 @@ import std/[options, tables, strutils, strformat, sugar] import jsony -import ../types/unifiedcard +import user, ../types/unifiedcard from ../../types import Card, CardKind, Video from ../../utils import twimg, https @@ -27,6 +27,14 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) result.text = data.topicDetail.title result.dest = "Topic" +proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) = + data.destination.parseDestination(card, result) + + result.kind = CardKind.jobDetails + result.title = data.title + result.text = data.shortDescriptionText + result.dest = &"@{data.profileUser.username} · {data.location}" + proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) = let app = card.appStoreData[data.appId][0] @@ -84,6 +92,8 @@ proc parseUnifiedCard*(json: string): Card = component.parseMedia(card, result) of buttonGroup: discard + of ComponentType.jobDetails: + component.data.parseJobDetails(card, result) of ComponentType.hidden: result.kind = CardKind.hidden of ComponentType.unknown: diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index 5962a87..07e0477 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -56,7 +56,7 @@ proc toUser*(raw: RawUser): User = tweets: raw.statusesCount, likes: raw.favouritesCount, media: raw.mediaCount, - verified: raw.verified or raw.extIsBlueVerified, + verifiedType: raw.verifiedType, protected: raw.protected, joinDate: parseTwitterDate(raw.createdAt), banner: getBanner(raw), @@ -68,6 +68,11 @@ proc toUser*(raw: RawUser): User = result.expandUserEntities(raw) +proc parseHook*(s: string; i: var int; v: var User) = + var u: RawUser + parseHook(s, i, u) + v = toUser u + proc parseUser*(json: string; username=""): User = handleErrors: case error.code @@ -75,7 +80,7 @@ proc parseUser*(json: string; username=""): User = of userNotFound: return else: echo "[error - parseUser]: ", error - result = toUser json.fromJson(RawUser) + result = json.fromJson(User) proc parseUsers*(json: string; after=""): Result[User] = result = Result[User](beginning: after.len == 0) diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index c30eed9..08100f9 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -1,5 +1,5 @@ import options -import user +from ../../types import User type GraphUser* = object @@ -9,7 +9,7 @@ type result*: UserResult UserResult = object - legacy*: RawUser + legacy*: User restId*: string isBlueVerified*: bool unavailableReason*: Option[string] diff --git a/src/experimental/types/guestaccount.nim b/src/experimental/types/guestaccount.nim new file mode 100644 index 0000000..244edb3 --- /dev/null +++ b/src/experimental/types/guestaccount.nim @@ -0,0 +1,4 @@ +type + RawAccount* = object + oauthToken*: string + oauthTokenSecret*: string diff --git a/src/experimental/types/timeline.nim b/src/experimental/types/timeline.nim index 28239ad..5ce6d9f 100644 --- a/src/experimental/types/timeline.nim +++ b/src/experimental/types/timeline.nim @@ -1,5 +1,5 @@ import std/tables -import user +from ../../types import User type Search* = object @@ -7,7 +7,7 @@ type timeline*: Timeline GlobalObjects = object - users*: Table[string, RawUser] + users*: Table[string, User] Timeline = object instructions*: seq[Instructions] diff --git a/src/experimental/types/unifiedcard.nim b/src/experimental/types/unifiedcard.nim index 6e83cad..e540a64 100644 --- a/src/experimental/types/unifiedcard.nim +++ b/src/experimental/types/unifiedcard.nim @@ -1,7 +1,10 @@ -import options, tables -from ../../types import VideoType, VideoVariant +import std/[options, tables, times] +import jsony +from ../../types import VideoType, VideoVariant, User type + Text* = distinct string + UnifiedCard* = object componentObjects*: Table[string, Component] destinationObjects*: Table[string, Destination] @@ -13,6 +16,7 @@ type media swipeableMedia buttonGroup + jobDetails appStoreDetails twitterListDetails communityDetails @@ -29,12 +33,15 @@ type appId*: string mediaId*: string destination*: string + location*: string title*: Text subtitle*: Text name*: Text memberCount*: int mediaList*: seq[MediaItem] topicDetail*: tuple[title: Text] + profileUser*: User + shortDescriptionText*: string MediaItem* = object id*: string @@ -69,12 +76,9 @@ type title*: Text category*: Text - Text = object - content: string - TypeField = Component | Destination | MediaEntity | AppStoreData -converter fromText*(text: Text): string = text.content +converter fromText*(text: Text): string = string(text) proc renameHook*(v: var TypeField; fieldName: var string) = if fieldName == "type": @@ -86,6 +90,7 @@ proc enumHook*(s: string; v: var ComponentType) = of "media": media of "swipeable_media": swipeableMedia of "button_group": buttonGroup + of "job_details": jobDetails of "app_store_details": appStoreDetails of "twitter_list_details": twitterListDetails of "community_details": communityDetails @@ -106,3 +111,18 @@ proc enumHook*(s: string; v: var MediaType) = of "photo": photo of "model3d": model3d else: echo "ERROR: Unknown enum value (MediaType): ", s; photo + +proc parseHook*(s: string; i: var int; v: var DateTime) = + var str: string + parseHook(s, i, str) + v = parse(str, "yyyy-MM-dd hh:mm:ss") + +proc parseHook*(s: string; i: var int; v: var Text) = + if s[i] == '"': + var str: string + parseHook(s, i, str) + v = Text(str) + else: + var t: tuple[content: string] + parseHook(s, i, t) + v = Text(t.content) diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim index 39331a0..7dc0194 100644 --- a/src/experimental/types/user.nim +++ b/src/experimental/types/user.nim @@ -1,5 +1,6 @@ import options import common +from ../../types import VerifiedType type RawUser* = object @@ -15,8 +16,7 @@ type favouritesCount*: int statusesCount*: int mediaCount*: int - verified*: bool - extIsBlueVerified*: bool + verifiedType*: VerifiedType protected*: bool profileLinkColor*: string profileBannerUrl*: string diff --git a/src/nitter.nim b/src/nitter.nim index 35c2372..f976db2 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -4,11 +4,10 @@ import config from net import Port from htmlgen import a from os import getEnv -from json import parseJson import jester -import types, config, prefs, formatters, redis_cache, http_pool, tokens +import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, @@ -19,9 +18,8 @@ const issuesUrl = "https://github.com/zedeus/nitter/issues" let accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") - accounts = parseJson(readFile(accountsPath)) -initAccountPool(cfg, parseJson(readFile(accountsPath))) +initAccountPool(cfg, accountsPath) if not cfg.enableDebug: # Silence Jester's query warning diff --git a/src/parser.nim b/src/parser.nim index 68ee078..fac0bd6 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -22,7 +22,7 @@ proc parseUser(js: JsonNode; id=""): User = tweets: js{"statuses_count"}.getInt, likes: js{"favourites_count"}.getInt, media: js{"media_count"}.getInt, - verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool, + verifiedType: parseEnum[VerifiedType](js{"verified_type"}.getStr("None")), protected: js{"protected"}.getBool, joinDate: js{"created_at"}.getTime ) @@ -35,8 +35,8 @@ proc parseGraphUser(js: JsonNode): User = user = ? js{"user_results", "result"} result = parseUser(user{"legacy"}) - if "is_blue_verified" in user: - result.verified = user{"is_blue_verified"}.getBool() + if result.verifiedType == VerifiedType.none and user{"is_blue_verified"}.getBool(false): + result.verifiedType = blue proc parseGraphList*(js: JsonNode): List = if js.isNull: return @@ -220,8 +220,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = ) ) - result.expandTweetEntities(js) - # fix for pinned threads if result.hasThread and result.threadId == 0: result.threadId = js{"self_thread", "id_str"}.getId @@ -258,6 +256,8 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = else: result.card = some parseCard(jsCard, js{"entities", "urls"}) + result.expandTweetEntities(js) + with jsMedia, js{"extended_entities", "media"}: for m in jsMedia: case m{"type"}.getStr @@ -442,6 +442,8 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet = return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.") of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}, isLegacy) + else: + discard if not js.hasKey("legacy"): return Tweet() @@ -592,8 +594,8 @@ proc parseGraphRetweetersTimeline*(js: JsonNode; root: string; after=""): UsersT proc parseGraphFollowTimeline*(js: JsonNode; root: string; after=""): UsersTimeline = return parseGraphUsersTimeline(js{"data", "user", "result", "timeline", "timeline"}, after) -proc parseGraphSearch*(js: JsonNode; after=""): Timeline = - result = Timeline(beginning: after.len == 0) +proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] = + result = Result[T](beginning: after.len == 0) let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"} if instructions.len == 0: @@ -604,13 +606,19 @@ proc parseGraphSearch*(js: JsonNode; after=""): Timeline = if typ == "TimelineAddEntries": for e in instruction{"entries"}: let entryId = e{"entryId"}.getStr - if entryId.startsWith("tweet"): - with tweetRes, e{"content", "itemContent", "tweet_results", "result"}: - let tweet = parseGraphTweet(tweetRes, true) - if not tweet.available: - tweet.id = parseBiggestInt(entryId.getId()) - result.content.add tweet - elif entryId.startsWith("cursor-bottom"): + when T is Tweets: + if entryId.startsWith("tweet"): + with tweetRes, e{"content", "itemContent", "tweet_results", "result"}: + let tweet = parseGraphTweet(tweetRes) + if not tweet.available: + tweet.id = parseBiggestInt(entryId.getId()) + result.content.add tweet + elif T is User: + if entryId.startsWith("user"): + with userRes, e{"content", "itemContent"}: + result.content.add parseGraphUser(userRes) + + if entryId.startsWith("cursor-bottom"): result.bottom = e{"content", "value"}.getStr elif typ == "TimelineReplaceEntry": if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"): diff --git a/src/parserutils.nim b/src/parserutils.nim index 7cf696e..00ea6f4 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -1,9 +1,17 @@ # SPDX-License-Identifier: AGPL-3.0-only -import std/[strutils, times, macros, htmlgen, options, algorithm, re] +import std/[times, macros, htmlgen, options, algorithm, re] +import std/strutils except escape import std/unicode except strip +from xmltree import escape import packedjson import types, utils, formatters +const + unicodeOpen = "\uFFFA" + unicodeClose = "\uFFFB" + xmlOpen = escape("<") + xmlClose = escape(">") + let unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})" unReplace = "$1@$2" @@ -238,7 +246,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) = .replacef(htRegex, htReplace) proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int]; - replyTo=""; hasQuote=false) = + replyTo=""; hasRedundantLink=false) = let hasCard = tweet.card.isSome var replacements = newSeq[ReplaceSlice]() @@ -249,7 +257,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic if urlStr.len == 0 or urlStr notin text: continue - replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote) + replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink) if hasCard and u{"url"}.getStr == get(tweet.card).url: get(tweet.card).url = u{"expanded_url"}.getStr @@ -289,9 +297,10 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = let entities = ? js{"entities"} - hasQuote = js{"is_quote_status"}.getBool textRange = js{"display_text_range"} textSlice = textRange{0}.getInt .. textRange{1}.getInt + hasQuote = js{"is_quote_status"}.getBool + hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails var replyTo = "" if tweet.replyId != 0: @@ -299,12 +308,14 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = replyTo = reply.getStr tweet.reply.add replyTo - tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote) + tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard) proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let entities = ? js{"entity_set"} - text = js{"text"}.getStr + text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose)) textSlice = 0..text.runeLen tweet.expandTextEntities(entities, text, textSlice) + + tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose)) diff --git a/src/query.nim b/src/query.nim index 49c5856..b5d79d9 100644 --- a/src/query.nim +++ b/src/query.nim @@ -67,7 +67,7 @@ proc genQueryParam*(query: Query): string = param &= "OR " if query.fromUser.len > 0 and query.kind in {posts, media}: - param &= "filter:self_threads OR-filter:replies " + param &= "filter:self_threads OR -filter:replies " if "nativeretweets" notin query.excludes: param &= "include:nativeretweets " diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 2387a42..1d77cca 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -52,6 +52,7 @@ proc initRedisPool*(cfg: Config) {.async.} = await migrate("profileDates", "p:*") await migrate("profileStats", "p:*") await migrate("userType", "p:*") + await migrate("verifiedType", "p:*") pool.withAcquire(r): # optimize memory usage for user ID buckets @@ -85,7 +86,7 @@ proc cache*(data: List) {.async.} = await setEx(data.listKey, listCacheTime, compress(toFlatty(data))) proc cache*(data: PhotoRail; name: string) {.async.} = - await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data))) + await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data))) proc cache*(data: User) {.async.} = if data.username.len == 0: return diff --git a/src/routes/debug.nim b/src/routes/debug.nim index 192786e..895a285 100644 --- a/src/routes/debug.nim +++ b/src/routes/debug.nim @@ -1,10 +1,13 @@ # SPDX-License-Identifier: AGPL-3.0-only import jester import router_utils -import ".."/[tokens, types] +import ".."/[auth, types] proc createDebugRouter*(cfg: Config) = router debug: - get "/.tokens": + get "/.health": + respJson getAccountPoolHealth() + + get "/.accounts": cond cfg.enableDebug - respJson getPoolJson() + respJson getAccountPoolDebug() diff --git a/src/routes/search.nim b/src/routes/search.nim index 676229e..e9f991d 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -29,7 +29,7 @@ proc createSearchRouter*(cfg: Config) = redirect("/" & q) var users: Result[User] try: - users = await getUserSearch(query, getCursor()) + users = await getGraphUserSearch(query, getCursor()) except InternalError: users = Result[User](beginning: true, query: query) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title) diff --git a/src/sass/include/_variables.scss b/src/sass/include/_variables.scss index 0f81235..0c95ff6 100644 --- a/src/sass/include/_variables.scss +++ b/src/sass/include/_variables.scss @@ -28,6 +28,8 @@ $more_replies_dots: #AD433B; $error_red: #420A05; $verified_blue: #1DA1F2; +$verified_business: #FAC82B; +$verified_government: #C1B6A4; $icon_text: $fg_color; $tab: $fg_color; diff --git a/src/sass/index.scss b/src/sass/index.scss index 9e2e347..6cab48e 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -39,6 +39,8 @@ body { --error_red: #{$error_red}; --verified_blue: #{$verified_blue}; + --verified_business: #{$verified_business}; + --verified_government: #{$verified_government}; --icon_text: #{$icon_text}; --tab: #{$fg_color}; @@ -141,17 +143,30 @@ ul { .verified-icon { color: var(--icon_text); - background-color: var(--verified_blue); border-radius: 50%; flex-shrink: 0; margin: 2px 0 3px 3px; - padding-top: 2px; - height: 12px; + padding-top: 3px; + height: 11px; width: 14px; font-size: 8px; display: inline-block; text-align: center; vertical-align: middle; + + &.blue { + background-color: var(--verified_blue); + } + + &.business { + color: var(--bg_panel); + background-color: var(--verified_business); + } + + &.government { + color: var(--bg_panel); + background-color: var(--verified_government); + } } @media(max-width: 600px) { diff --git a/src/sass/navbar.scss b/src/sass/navbar.scss index cf9c80e..47a8765 100644 --- a/src/sass/navbar.scss +++ b/src/sass/navbar.scss @@ -70,8 +70,9 @@ nav { .lp { height: 14px; - margin-top: 2px; - display: block; + display: inline-block; + position: relative; + top: 2px; fill: var(--fg_nav); &:hover { diff --git a/src/sass/profile/card.scss b/src/sass/profile/card.scss index 85878e4..46a9679 100644 --- a/src/sass/profile/card.scss +++ b/src/sass/profile/card.scss @@ -115,7 +115,7 @@ } .profile-card-tabs-name { - @include breakable; + flex-shrink: 100; } .profile-card-avatar { diff --git a/src/sass/search.scss b/src/sass/search.scss index 0311fb0..f70f7ea 100644 --- a/src/sass/search.scss +++ b/src/sass/search.scss @@ -14,6 +14,8 @@ button { margin: 0 2px 0 0; height: 23px; + display: flex; + align-items: center; } .pref-input { diff --git a/src/types.nim b/src/types.nim index 74fd677..5e58716 100644 --- a/src/types.nim +++ b/src/types.nim @@ -10,16 +10,13 @@ type BadClientError* = object of CatchableError TimelineKind* {.pure.} = enum - tweets - replies - media + tweets, replies, media Api* {.pure.} = enum tweetDetail tweetResult photoRail search - userSearch list listBySlug listMembers @@ -42,7 +39,7 @@ type limitedAt*: int GuestAccount* = ref object - id*: string + id*: int64 oauthToken*: string oauthSecret*: string pending*: int @@ -69,6 +66,12 @@ type tweetUnavailable = 421 tweetCensored = 422 + VerifiedType* = enum + none = "None" + blue = "Blue" + business = "Business" + government = "Government" + User* = object id*: string username*: string @@ -84,7 +87,7 @@ type tweets*: int likes*: int media*: int - verified*: bool + verifiedType*: VerifiedType protected*: bool suspended*: bool joinDate*: DateTime @@ -168,9 +171,10 @@ type imageDirectMessage = "image_direct_message" audiospace = "audiospace" newsletterPublication = "newsletter_publication" + jobDetails = "job_details" hidden unknown - + Card* = object kind*: CardKind url*: string diff --git a/src/utils.nim b/src/utils.nim index 9002bbf..c96a6dd 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -16,7 +16,8 @@ const "twimg.com", "abs.twimg.com", "pbs.twimg.com", - "video.twimg.com" + "video.twimg.com", + "x.com" ] proc setHmacKey*(key: string) = @@ -57,4 +58,4 @@ proc isTwitterUrl*(uri: Uri): bool = uri.hostname in twitterDomains proc isTwitterUrl*(url: string): bool = - parseUri(url).hostname in twitterDomains + isTwitterUrl(parseUri(url)) diff --git a/src/views/general.nim b/src/views/general.nim index 5e96d02..87d30f2 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=18") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=19") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2") if theme.len > 0: diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 9dffdcb..f298fad 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -23,6 +23,13 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = if text.len > 0: text " " & text +template verifiedIcon*(user: User): untyped {.dirty.} = + if user.verifiedType != VerifiedType.none: + let lower = ($user.verifiedType).toLowerAscii() + icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account") + else: + text "" + proc linkUser*(user: User, class=""): VNode = let isName = "username" notin class @@ -32,11 +39,11 @@ proc linkUser*(user: User, class=""): VNode = buildHtml(a(href=href, class=class, title=nameText)): text nameText - if isName and user.verified: - icon "ok", class="verified-icon", title="Verified account" - if isName and user.protected: - text " " - icon "lock", title="Protected account" + if isName: + verifiedIcon(user) + if user.protected: + text " " + icon "lock", title="Protected account" proc linkText*(text: string; class=""): VNode = let url = if "http" notin text: https & text else: text diff --git a/src/views/tweet.nim b/src/views/tweet.nim index b178edc..13b4a24 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -205,8 +205,7 @@ proc renderAttribution(user: User; prefs: Prefs): VNode = buildHtml(a(class="attribution", href=("/" & user.username))): renderMiniAvatar(user, prefs) strong: text user.fullname - if user.verified: - icon "ok", class="verified-icon", title="Verified account" + verifiedIcon(user) proc renderMediaTags(tags: seq[User]): VNode = buildHtml(tdiv(class="media-tag-block")): diff --git a/tests/test_card.py b/tests/test_card.py index 733bd40..8da91a2 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -21,9 +21,9 @@ card = [ no_thumb = [ ['FluentAI/status/1116417904831029248', - 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', - 'One of the only ways to improve Alexa is to have human beings check it for errors', - 'theverge.com'], + 'LinkedIn', + 'This link will take you to a page that’s not on LinkedIn', + 'lnkd.in'], ['Thom_Wolf/status/1122466524860702729', 'facebookresearch/fairseq', diff --git a/tests/test_quote.py b/tests/test_quote.py index 1b458ea..4921c21 100644 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -9,7 +9,7 @@ text = [ What are we doing wrong? reuters.com/article/us-norwa…"""], ['nim_lang/status/1491461266849808397#m', - 'Nim language', '@nim_lang', + 'Nim', '@nim_lang', """What's better than Nim 1.6.0? Nim 1.6.2 :) diff --git a/tests/test_tweet.py b/tests/test_tweet.py index 7a3c4ed..839e6c5 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -1,4 +1,4 @@ -from base import BaseTestCase, Tweet, get_timeline_tweet +from base import BaseTestCase, Tweet, Conversation, get_timeline_tweet from parameterized import parameterized # image = tweet + 'div.attachments.media-body > div > div > a > div > img' @@ -35,7 +35,16 @@ multiline = [ CALM AND CLICHÉ - ON"""] + ON"""], + [1718660434457239868, 'WebDesignMuseum', + """ +Happy 32nd Birthday HTML tags! + +On October 29, 1991, the internet pioneer, Tim Berners-Lee, published a document entitled HTML Tags. + +The document contained a description of the first 18 HTML tags: , <nextid>, <a>, <isindex>, <plaintext>, <listing>, <p>, <h1>…<h6>, <address>, <hp1>, <hp2>…, <dl>, <dt>, <dd>, <ul>, <li>,<menu> and <dir>. The design of the first version of HTML language was influenced by the SGML universal markup language. + +#WebDesignHistory"""] ] link = [ @@ -74,22 +83,18 @@ retweet = [ [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr'] ] -# reply = [ -# ['mobile_test/with_replies', 15] -# ] - class TweetTest(BaseTestCase): - # @parameterized.expand(timeline) - # def test_timeline(self, index, fullname, username, date, tid, text): - # self.open_nitter(username) - # tweet = get_timeline_tweet(index) - # self.assert_exact_text(fullname, tweet.fullname) - # self.assert_exact_text('@' + username, tweet.username) - # self.assert_exact_text(date, tweet.date) - # self.assert_text(text, tweet.text) - # permalink = self.find_element(tweet.date + ' a') - # self.assertIn(tid, permalink.get_attribute('href')) + @parameterized.expand(timeline) + def test_timeline(self, index, fullname, username, date, tid, text): + self.open_nitter(username) + tweet = get_timeline_tweet(index) + self.assert_exact_text(fullname, tweet.fullname) + self.assert_exact_text('@' + username, tweet.username) + self.assert_exact_text(date, tweet.date) + self.assert_text(text, tweet.text) + permalink = self.find_element(tweet.date + ' a') + self.assertIn(tid, permalink.get_attribute('href')) @parameterized.expand(status) def test_status(self, tid, fullname, username, date, text): @@ -103,18 +108,18 @@ class TweetTest(BaseTestCase): @parameterized.expand(multiline) def test_multiline_formatting(self, tid, username, text): self.open_nitter(f'{username}/status/{tid}') - self.assert_text(text.strip('\n'), '.main-tweet') + self.assert_text(text.strip('\n'), Conversation.main) @parameterized.expand(emoji) def test_emoji(self, tweet, text): self.open_nitter(tweet) - self.assert_text(text, '.main-tweet') + self.assert_text(text, Conversation.main) @parameterized.expand(link) def test_link(self, tweet, links): self.open_nitter(tweet) for link in links: - self.assert_text(link, '.main-tweet') + self.assert_text(link, Conversation.main) @parameterized.expand(username) def test_username(self, tweet, usernames): @@ -123,22 +128,22 @@ class TweetTest(BaseTestCase): link = self.find_link_text(f'@{un}') self.assertIn(f'/{un}', link.get_property('href')) - # @parameterized.expand(retweet) - # def test_retweet(self, index, url, retweet_by, fullname, username, text): - # self.open_nitter(url) - # tweet = get_timeline_tweet(index) - # self.assert_text(f'{retweet_by} retweeted', tweet.retweet) - # self.assert_text(text, tweet.text) - # self.assert_exact_text(fullname, tweet.fullname) - # self.assert_exact_text(username, tweet.username) + @parameterized.expand(retweet) + def test_retweet(self, index, url, retweet_by, fullname, username, text): + self.open_nitter(url) + tweet = get_timeline_tweet(index) + self.assert_text(f'{retweet_by} retweeted', tweet.retweet) + self.assert_text(text, tweet.text) + self.assert_exact_text(fullname, tweet.fullname) + self.assert_exact_text(username, tweet.username) @parameterized.expand(invalid) def test_invalid_id(self, tweet): self.open_nitter(tweet) self.assert_text('Tweet not found', '.error-panel') - # @parameterized.expand(reply) - # def test_thread(self, tweet, num): - # self.open_nitter(tweet) - # thread = self.find_element(f'.timeline > div:nth-child({num})') - # self.assertIn(thread.get_attribute('class'), 'thread-line') + #@parameterized.expand(reply) + #def test_thread(self, tweet, num): + #self.open_nitter(tweet) + #thread = self.find_element(f'.timeline > div:nth-child({num})') + #self.assertIn(thread.get_attribute('class'), 'thread-line')