From 7abcb489f4176532322669e97eb2503a78b2bbc2 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 18 Sep 2023 17:15:09 +0000 Subject: [PATCH 01/31] Increase photo rail cache ttl --- src/redis_cache.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 2387a42..a8b5ff8 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -85,7 +85,7 @@ proc cache*(data: List) {.async.} = await setEx(data.listKey, listCacheTime, compress(toFlatty(data))) proc cache*(data: PhotoRail; name: string) {.async.} = - await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data))) + await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data))) proc cache*(data: User) {.async.} = if data.username.len == 0: return From 7d147899103bf1a21b291641fd4230a02ba48fab Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 18 Sep 2023 18:24:23 +0000 Subject: [PATCH 02/31] Improve guest accounts loading, add JSONL support --- .gitignore | 1 + src/experimental/parser/guestaccount.nim | 20 ++++++++++++++++++++ src/experimental/types/guestaccount.nim | 4 ++++ src/nitter.nim | 4 +--- src/tokens.nim | 23 +++++++++++++++-------- 5 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 src/experimental/parser/guestaccount.nim create mode 100644 src/experimental/types/guestaccount.nim diff --git a/.gitignore b/.gitignore index d43cc3f..ea520dc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ nitter /public/css/style.css /public/md/*.html nitter.conf +guest_accounts.json* dump.rdb diff --git a/src/experimental/parser/guestaccount.nim b/src/experimental/parser/guestaccount.nim new file mode 100644 index 0000000..4d8ff47 --- /dev/null +++ b/src/experimental/parser/guestaccount.nim @@ -0,0 +1,20 @@ +import jsony +import ../types/guestaccount +from ../../types import GuestAccount + +proc toGuestAccount(account: RawAccount): GuestAccount = + let id = account.oauthToken[0 ..< account.oauthToken.find('-')] + result = GuestAccount( + id: id, + oauthToken: account.oauthToken, + oauthSecret: account.oauthTokenSecret + ) + +proc parseGuestAccount*(raw: string): GuestAccount = + let rawAccount = raw.fromJson(RawAccount) + result = rawAccount.toGuestAccount + +proc parseGuestAccounts*(path: string): seq[GuestAccount] = + let rawAccounts = readFile(path).fromJson(seq[RawAccount]) + for account in rawAccounts: + result.add account.toGuestAccount diff --git a/src/experimental/types/guestaccount.nim b/src/experimental/types/guestaccount.nim new file mode 100644 index 0000000..244edb3 --- /dev/null +++ b/src/experimental/types/guestaccount.nim @@ -0,0 +1,4 @@ +type + RawAccount* = object + oauthToken*: string + oauthTokenSecret*: string diff --git a/src/nitter.nim b/src/nitter.nim index 4a4ec13..1b4862b 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -3,7 +3,6 @@ import asyncdispatch, strformat, logging from net import Port from htmlgen import a from os import getEnv -from json import parseJson import jester @@ -21,9 +20,8 @@ let (cfg, fullCfg) = getConfig(configPath) accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") - accounts = parseJson(readFile(accountsPath)) -initAccountPool(cfg, parseJson(readFile(accountsPath))) +initAccountPool(cfg, accountsPath) if not cfg.enableDebug: # Silence Jester's query warning diff --git a/src/tokens.nim b/src/tokens.nim index 3e20597..b8a50e3 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -1,6 +1,7 @@ #SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, times, json, random, strutils, tables, sets +import asyncdispatch, times, json, random, strutils, tables, sets, os import types +import experimental/parser/guestaccount # max requests at a time per account to avoid race conditions const @@ -141,12 +142,18 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = account.apis[api] = RateLimit(remaining: remaining, reset: reset) -proc initAccountPool*(cfg: Config; accounts: JsonNode) = +proc initAccountPool*(cfg: Config; path: string) = enableLogging = cfg.enableDebug - for account in accounts: - accountPool.add GuestAccount( - id: account{"user", "id_str"}.getStr, - oauthToken: account{"oauth_token"}.getStr, - oauthSecret: account{"oauth_token_secret"}.getStr, - ) + let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path + + if fileExists(jsonlPath): + log "Parsing JSONL guest accounts file: ", jsonlPath + for line in jsonlPath.lines: + accountPool.add parseGuestAccount(line) + elif fileExists(path): + log "Parsing JSON guest accounts file: ", path + accountPool = parseGuestAccounts(path) + else: + echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." + quit 1 From 537af7fd5e846e614315fbbff8741d3c7f0c74ba Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 19 Sep 2023 01:29:41 +0000 Subject: [PATCH 03/31] Improve Liberapay css for Firefox compatibility --- src/sass/navbar.scss | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sass/navbar.scss b/src/sass/navbar.scss index cf9c80e..47a8765 100644 --- a/src/sass/navbar.scss +++ b/src/sass/navbar.scss @@ -70,8 +70,9 @@ nav { .lp { height: 14px; - margin-top: 2px; - display: block; + display: inline-block; + position: relative; + top: 2px; fill: var(--fg_nav); &:hover { From 735b30c2da336cc57be2b98c48a6f7e826fdaed0 Mon Sep 17 00:00:00 2001 From: LS <66217791+DrSocket@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:13:06 +0100 Subject: [PATCH 04/31] fix(nitter): add graphql user search (#1047) * fix(nitter): add graphql user search * fix(nitter): rm gitignore 2nd guest_accounts * fix(nitter): keep query from user search in result. remove personal mods * fix(nitter): removce useless line gitignore --- src/api.nim | 28 ++++++++++++++++------------ src/consts.nim | 3 +-- src/parser.nim | 24 +++++++++++++++--------- src/routes/search.nim | 2 +- src/tokens.nim | 1 - src/types.nim | 1 - 6 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/api.nim b/src/api.nim index 4ac999c..d6a4564 100644 --- a/src/api.nim +++ b/src/api.nim @@ -112,25 +112,29 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = if after.len > 0: variables["cursor"] = % after let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphSearch(await fetch(url, Api.search), after) + result = parseGraphSearch[Tweets](await fetch(url, Api.search), after) result.query = query -proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = +proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} = if query.text.len == 0: return Result[User](query: query, beginning: true) - let - page = if page.len == 0: "1" else: page - url = userSearch ? genParams({"q": query.text, "skip_status": "1", "page": page}) - js = await fetchRaw(url, Api.userSearch) - - result = parseUsers(js) + var + variables = %*{ + "rawQuery": query.text, + "count": 20, + "product": "People", + "withDownvotePerspective": false, + "withReactionsMetadata": false, + "withReactionsPerspective": false + } + if after.len > 0: + variables["cursor"] = % after + result.beginning = false + let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} + result = parseGraphSearch[User](await fetch(url, Api.search), after) result.query = query - if page.len == 0: - result.bottom = "2" - elif page.allCharsInSet(Digits): - result.bottom = $(parseInt(page) + 1) proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = if name.len == 0: return diff --git a/src/consts.nim b/src/consts.nim index 96cea47..d3a3d80 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -9,7 +9,6 @@ const activate* = $(api / "1.1/guest/activate.json") photoRail* = api / "1.1/statuses/media_timeline.json" - userSearch* = api / "1.1/users/search.json" graphql = api / "graphql" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" @@ -35,7 +34,7 @@ const "include_user_entities": "1", "include_ext_reply_count": "1", "include_ext_is_blue_verified": "1", - #"include_ext_verified_type": "1", + # "include_ext_verified_type": "1", "include_ext_media_color": "0", "cards_platform": "Web-13", "tweet_mode": "extended", diff --git a/src/parser.nim b/src/parser.nim index 776f176..cebf6f1 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -443,8 +443,8 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = tweet.id = parseBiggestInt(entryId) result.pinned = some tweet -proc parseGraphSearch*(js: JsonNode; after=""): Timeline = - result = Timeline(beginning: after.len == 0) +proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] = + result = Result[T](beginning: after.len == 0) let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"} if instructions.len == 0: @@ -455,13 +455,19 @@ proc parseGraphSearch*(js: JsonNode; after=""): Timeline = if typ == "TimelineAddEntries": for e in instruction{"entries"}: let entryId = e{"entryId"}.getStr - if entryId.startsWith("tweet"): - with tweetRes, e{"content", "itemContent", "tweet_results", "result"}: - let tweet = parseGraphTweet(tweetRes, true) - if not tweet.available: - tweet.id = parseBiggestInt(entryId.getId()) - result.content.add tweet - elif entryId.startsWith("cursor-bottom"): + when T is Tweets: + if entryId.startsWith("tweet"): + with tweetRes, e{"content", "itemContent", "tweet_results", "result"}: + let tweet = parseGraphTweet(tweetRes) + if not tweet.available: + tweet.id = parseBiggestInt(entryId.getId()) + result.content.add tweet + elif T is User: + if entryId.startsWith("user"): + with userRes, e{"content", "itemContent"}: + result.content.add parseGraphUser(userRes) + + if entryId.startsWith("cursor-bottom"): result.bottom = e{"content", "value"}.getStr elif typ == "TimelineReplaceEntry": if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"): diff --git a/src/routes/search.nim b/src/routes/search.nim index 676229e..e9f991d 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -29,7 +29,7 @@ proc createSearchRouter*(cfg: Config) = redirect("/" & q) var users: Result[User] try: - users = await getUserSearch(query, getCursor()) + users = await getGraphUserSearch(query, getCursor()) except InternalError: users = Result[User](beginning: true, query: query) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title) diff --git a/src/tokens.nim b/src/tokens.nim index b8a50e3..ca74ddc 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -62,7 +62,6 @@ proc getPoolJson*(): JsonNode = Api.userRestId, Api.userScreenName, Api.tweetResult, Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500 - of Api.userSearch: 900 reqs = maxReqs - apiStatus.remaining reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs diff --git a/src/types.nim b/src/types.nim index 4cacc4b..3f5f8ac 100644 --- a/src/types.nim +++ b/src/types.nim @@ -19,7 +19,6 @@ type tweetResult photoRail search - userSearch list listBySlug listMembers From 32e3469e3a580464a79c6b2b6bfbaa6757bd8cfe Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 05:53:55 +0000 Subject: [PATCH 05/31] Fix multi-user timelines --- src/query.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query.nim b/src/query.nim index d128f6f..06e1da2 100644 --- a/src/query.nim +++ b/src/query.nim @@ -60,7 +60,7 @@ proc genQueryParam*(query: Query): string = param &= "OR " if query.fromUser.len > 0 and query.kind in {posts, media}: - param &= "filter:self_threads OR-filter:replies " + param &= "filter:self_threads OR -filter:replies " if "nativeretweets" notin query.excludes: param &= "include:nativeretweets " From edad09f4c934fb44da008256994ba40347b0f9c9 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 08:31:51 +0000 Subject: [PATCH 06/31] Update nimcrypto and jsony --- nitter.nimble | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index e6a1909..20aab81 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -14,7 +14,7 @@ requires "nim >= 1.4.8" requires "jester#baca3f" requires "karax#5cf360c" requires "sass#7dfdd03" -requires "nimcrypto#4014ef9" +requires "nimcrypto#a079df9" requires "markdown#158efe3" requires "packedjson#9e6fbb6" requires "supersnappy#6c94198" @@ -22,7 +22,7 @@ requires "redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#ca5989a" requires "flatty#e668085" -requires "jsony#ea811be" +requires "jsony#1de1f08" requires "oauth#b8c163b" # Tasks From 089275826cae70c30183e1bd21b49538e43bd4d9 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 11:32:21 +0000 Subject: [PATCH 07/31] Bump minimum Nim version --- config.nims | 7 +------ nitter.nimble | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/config.nims b/config.nims index b7e52d0..4a7af27 100644 --- a/config.nims +++ b/config.nims @@ -7,12 +7,7 @@ # disable annoying warnings warning("GcUnsafe2", off) +warning("HoleEnumConv", off) hint("XDeclaredButNotUsed", off) hint("XCannotRaiseY", off) hint("User", off) - -const - nimVersion = (major: NimMajor, minor: NimMinor, patch: NimPatch) - -when nimVersion >= (1, 6, 0): - warning("HoleEnumConv", off) diff --git a/nitter.nimble b/nitter.nimble index 20aab81..37f9229 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -10,7 +10,7 @@ bin = @["nitter"] # Dependencies -requires "nim >= 1.4.8" +requires "nim >= 1.6.10" requires "jester#baca3f" requires "karax#5cf360c" requires "sass#7dfdd03" From 412055864940a79d3203aa41568c6ed67c5bb5f8 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 12:04:32 +0000 Subject: [PATCH 08/31] Replace /.tokens with /.health and /.accounts --- nitter.example.conf | 2 +- src/apiutils.nim | 4 +- src/{tokens.nim => auth.nim} | 113 +++++++++++++++-------- src/experimental/parser/guestaccount.nim | 3 +- src/nitter.nim | 2 +- src/routes/debug.nim | 9 +- src/types.nim | 2 +- 7 files changed, 86 insertions(+), 49 deletions(-) rename src/{tokens.nim => auth.nim} (69%) diff --git a/nitter.example.conf b/nitter.example.conf index 0d4deb7..f0b4214 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -23,7 +23,7 @@ redisMaxConnections = 30 hmacKey = "secretkey" # random key for cryptographic signing of video urls base64Media = false # use base64 encoding for proxied media urls enableRSS = true # set this to false to disable RSS feeds -enableDebug = false # enable request logs and debug endpoints (/.tokens) +enableDebug = false # enable request logs and debug endpoints (/.accounts) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" tokenCount = 10 diff --git a/src/apiutils.nim b/src/apiutils.nim index 9ac101e..1ff05eb 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import jsony, packedjson, zippy, oauth1 -import types, tokens, consts, parserutils, http_pool +import types, auth, consts, parserutils, http_pool import experimental/types/common const @@ -120,7 +120,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = except OSError as e: raise e except Exception as e: - let id = if account.isNil: "null" else: account.id + let id = if account.isNil: "null" else: $account.id echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url raise rateLimitError() finally: diff --git a/src/tokens.nim b/src/auth.nim similarity index 69% rename from src/tokens.nim rename to src/auth.nim index ca74ddc..560fb84 100644 --- a/src/tokens.nim +++ b/src/auth.nim @@ -1,5 +1,5 @@ #SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, times, json, random, strutils, tables, sets, os +import asyncdispatch, times, json, random, strutils, tables, intsets, os import types import experimental/parser/guestaccount @@ -7,6 +7,21 @@ import experimental/parser/guestaccount const maxConcurrentReqs = 2 dayInSeconds = 24 * 60 * 60 + apiMaxReqs: Table[Api, int] = { + Api.search: 50, + Api.tweetDetail: 150, + Api.photoRail: 180, + Api.userTweets: 500, + Api.userTweetsAndReplies: 500, + Api.userMedia: 500, + Api.userRestId: 500, + Api.userScreenName: 500, + Api.tweetResult: 500, + Api.list: 500, + Api.listTweets: 500, + Api.listMembers: 500, + Api.listBySlug: 500 + }.toTable var accountPool: seq[GuestAccount] @@ -15,20 +30,64 @@ var template log(str: varargs[string, `$`]) = if enableLogging: echo "[accounts] ", str.join("") -proc getPoolJson*(): JsonNode = - var - list = newJObject() - totalReqs = 0 - totalPending = 0 - limited: HashSet[string] - reqsPerApi: Table[string, int] - +proc getAccountPoolHealth*(): JsonNode = let now = epochTime().int - for account in accountPool: - totalPending.inc(account.pending) + var + totalReqs = 0 + limited: IntSet + reqsPerApi: Table[string, int] + oldest = now + newest = 0 + average = 0 - var includeAccount = false + for account in accountPool: + # Twitter snowflake conversion + let created = ((account.id shr 22) + 1288834974657) div 1000 + + if created > newest: + newest = created + if created < oldest: + oldest = created + average.inc created + + for api in account.apis.keys: + let + apiStatus = account.apis[api] + reqs = apiMaxReqs[api] - apiStatus.remaining + + reqsPerApi.mgetOrPut($api, 0).inc reqs + totalReqs.inc reqs + + if apiStatus.limited: + limited.incl account.id + + if accountPool.len > 0: + average = average div accountPool.len + else: + oldest = 0 + average = 0 + + return %*{ + "accounts": %*{ + "total": accountPool.len, + "active": accountPool.len - limited.card, + "limited": limited.card, + "oldest": $fromUnix(oldest), + "newest": $fromUnix(newest), + "average": $fromUnix(average) + }, + "requests": %*{ + "total": totalReqs, + "apis": reqsPerApi + } + } + +proc getAccountPoolDebug*(): JsonNode = + let now = epochTime().int + var list = newJObject() + + for account in accountPool: let accountJson = %*{ "apis": newJObject(), "pending": account.pending, @@ -47,37 +106,11 @@ proc getPoolJson*(): JsonNode = if apiStatus.limited: obj["limited"] = %true - limited.incl account.id accountJson{"apis", $api} = obj - includeAccount = true + list[$account.id] = accountJson - let - maxReqs = - case api - of Api.search: 50 - of Api.tweetDetail: 150 - of Api.photoRail: 180 - of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, - Api.userRestId, Api.userScreenName, - Api.tweetResult, - Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500 - reqs = maxReqs - apiStatus.remaining - - reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs - totalReqs.inc(reqs) - - if includeAccount: - list[account.id] = accountJson - - return %*{ - "amount": accountPool.len, - "limited": limited.card, - "requests": totalReqs, - "pending": totalPending, - "apis": reqsPerApi, - "accounts": list - } + return %list proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") diff --git a/src/experimental/parser/guestaccount.nim b/src/experimental/parser/guestaccount.nim index 4d8ff47..f7e6d34 100644 --- a/src/experimental/parser/guestaccount.nim +++ b/src/experimental/parser/guestaccount.nim @@ -1,3 +1,4 @@ +import std/strutils import jsony import ../types/guestaccount from ../../types import GuestAccount @@ -5,7 +6,7 @@ from ../../types import GuestAccount proc toGuestAccount(account: RawAccount): GuestAccount = let id = account.oauthToken[0 ..< account.oauthToken.find('-')] result = GuestAccount( - id: id, + id: parseBiggestInt(id), oauthToken: account.oauthToken, oauthSecret: account.oauthTokenSecret ) diff --git a/src/nitter.nim b/src/nitter.nim index 1b4862b..dfc1dfd 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -6,7 +6,7 @@ from os import getEnv import jester -import types, config, prefs, formatters, redis_cache, http_pool, tokens +import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, diff --git a/src/routes/debug.nim b/src/routes/debug.nim index 192786e..895a285 100644 --- a/src/routes/debug.nim +++ b/src/routes/debug.nim @@ -1,10 +1,13 @@ # SPDX-License-Identifier: AGPL-3.0-only import jester import router_utils -import ".."/[tokens, types] +import ".."/[auth, types] proc createDebugRouter*(cfg: Config) = router debug: - get "/.tokens": + get "/.health": + respJson getAccountPoolHealth() + + get "/.accounts": cond cfg.enableDebug - respJson getPoolJson() + respJson getAccountPoolDebug() diff --git a/src/types.nim b/src/types.nim index 3f5f8ac..3b0d55c 100644 --- a/src/types.nim +++ b/src/types.nim @@ -36,7 +36,7 @@ type limitedAt*: int GuestAccount* = ref object - id*: string + id*: BiggestInt oauthToken*: string oauthSecret*: string pending*: int From b62d73dbd373f08af07c7a79efcd790d3bc1a49c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Tue, 31 Oct 2023 23:33:08 +0100 Subject: [PATCH 09/31] nim version min require + update dockerfile arm (#1053) --- Dockerfile.arm64 | 6 +++--- nitter.nimble | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 6cd6744..fbad812 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1,7 +1,7 @@ -FROM alpine:3.17 as nim +FROM alpine:3.18 as nim LABEL maintainer="setenforce@protonmail.com" -RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre +RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.14-r0" nimble pcre WORKDIR /src/nitter @@ -13,7 +13,7 @@ RUN nimble build -d:danger -d:lto -d:strip \ && nimble scss \ && nimble md -FROM alpine:3.17 +FROM alpine:3.18 WORKDIR /src/ RUN apk --no-cache add ca-certificates pcre openssl1.1-compat COPY --from=nim /src/nitter/nitter ./ diff --git a/nitter.nimble b/nitter.nimble index 7771b31..3a490a5 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -10,7 +10,7 @@ bin = @["nitter"] # Dependencies -requires "nim >= 1.4.8" +requires "nim >= 1.6.10" requires "jester#baca3f" requires "karax#5cf360c" requires "sass#7dfdd03" From b8103cf5010ea515c3b9722dbda19f374c6ebdaa Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 23:02:45 +0000 Subject: [PATCH 10/31] Fix compilation on Nim 1.6.x --- src/auth.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index 560fb84..97ee301 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,5 +1,5 @@ #SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, times, json, random, strutils, tables, intsets, os +import std/[asyncdispatch, times, json, random, strutils, tables, packedsets, os] import types import experimental/parser/guestaccount @@ -35,21 +35,21 @@ proc getAccountPoolHealth*(): JsonNode = var totalReqs = 0 - limited: IntSet + limited: PackedSet[BiggestInt] reqsPerApi: Table[string, int] - oldest = now - newest = 0 - average = 0 + oldest = now.int64 + newest = 0'i64 + average = 0'i64 for account in accountPool: # Twitter snowflake conversion - let created = ((account.id shr 22) + 1288834974657) div 1000 + let created = int64(((account.id shr 22) + 1288834974657) div 1000) if created > newest: newest = created if created < oldest: oldest = created - average.inc created + average += created for api in account.apis.keys: let From 60a82563da979f81c24eb51b7ae031f4086c03fd Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 23:46:24 +0000 Subject: [PATCH 11/31] Run tests on multiple Nim versions --- .github/workflows/run-tests.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 37979cb..0af20a3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -11,6 +11,12 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + matrix: + nim: + - "1.6.10" + - "1.6.x" + - "2.0.x" steps: - uses: actions/checkout@v3 with: @@ -28,7 +34,8 @@ jobs: cache: "pip" - uses: jiro4989/setup-nim-action@v1 with: - nim-version: "1.x" + nim-version: ${{ matrix.nim }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - run: nimble build -d:release -Y - run: pip install seleniumbase - run: seleniumbase install chromedriver From b930a3d5bf4b0ce679b5086ae712b65f279f1a49 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 23:54:11 +0000 Subject: [PATCH 12/31] Fix guest accounts CI setup --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0af20a3..3fbe64a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -50,6 +50,6 @@ jobs: env: GUEST_ACCOUNTS: ${{ secrets.GUEST_ACCOUNTS }} run: | - echo $GUEST_ACCOUNTS > ./guest_accounts.json + echo $GUEST_ACCOUNTS > ./guest_accounts.jsonl ./nitter & pytest -n4 tests From 33bad37128abd3987e823d32e2ef90ea91f8e4f5 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 01:24:51 +0000 Subject: [PATCH 13/31] Fix guest accounts CI setup attempt 2 --- .github/workflows/run-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 3fbe64a..12adb1f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -44,12 +44,11 @@ jobs: run: | sudo apt install libsass-dev -y cp nitter.example.conf nitter.conf + sed -i 's/enableDebug = false/enableDebug = true/g' nimble md nimble scss + echo "${{ env.GUEST_ACCOUNTS }}" > ./guest_accounts.jsonl - name: Run tests - env: - GUEST_ACCOUNTS: ${{ secrets.GUEST_ACCOUNTS }} run: | - echo $GUEST_ACCOUNTS > ./guest_accounts.jsonl ./nitter & pytest -n4 tests From 006b91c90391e972b8e1377df18f7f1da5718c87 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 04:04:45 +0000 Subject: [PATCH 14/31] Prevent annoying warnings on devel --- src/parser.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parser.nim b/src/parser.nim index cebf6f1..fcee13f 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -323,6 +323,8 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet = return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.") of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}, isLegacy) + else: + discard if not js.hasKey("legacy"): return Tweet() From b0b335106d992acdca2d82da82fe9dee89044404 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 04:06:42 +0000 Subject: [PATCH 15/31] Fix missing CI file argument --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 12adb1f..f9df235 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -44,7 +44,7 @@ jobs: run: | sudo apt install libsass-dev -y cp nitter.example.conf nitter.conf - sed -i 's/enableDebug = false/enableDebug = true/g' + sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf nimble md nimble scss echo "${{ env.GUEST_ACCOUNTS }}" > ./guest_accounts.jsonl From 58e73a14c576b275dde9f9af8d4f31f0f6202255 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 04:13:22 +0000 Subject: [PATCH 16/31] Fix guest accounts CI setup attempt 3 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f9df235..948d284 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -47,7 +47,7 @@ jobs: sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf nimble md nimble scss - echo "${{ env.GUEST_ACCOUNTS }}" > ./guest_accounts.jsonl + echo "${{ vars.GUEST_ACCOUNTS }}" > ./guest_accounts.jsonl - name: Run tests run: | ./nitter & From 1d20bd01cb9db816e47b1911f9836c442f2726c9 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 04:16:26 +0000 Subject: [PATCH 17/31] Remove redundant "active" field from /.health --- src/auth.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/src/auth.nim b/src/auth.nim index 97ee301..8be435c 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -71,7 +71,6 @@ proc getAccountPoolHealth*(): JsonNode = return %*{ "accounts": %*{ "total": accountPool.len, - "active": accountPool.len - limited.card, "limited": limited.card, "oldest": $fromUnix(oldest), "newest": $fromUnix(newest), From 7b3fcdc622628febc306cf9c0d4a33c493d690db Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 04:19:10 +0000 Subject: [PATCH 18/31] Fix guest accounts CI setup attempt 4 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 948d284..76af329 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -47,7 +47,7 @@ jobs: sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf nimble md nimble scss - echo "${{ vars.GUEST_ACCOUNTS }}" > ./guest_accounts.jsonl + echo '${{ secrets.GUEST_ACCOUNTS }}' > ./guest_accounts.jsonl - name: Run tests run: | ./nitter & From 623424f5160b3e99e6e4b9675c8705f87fbf9d72 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 04:52:44 +0000 Subject: [PATCH 19/31] Fix outdated test --- tests/test_card.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_card.py b/tests/test_card.py index 733bd40..8da91a2 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -21,9 +21,9 @@ card = [ no_thumb = [ ['FluentAI/status/1116417904831029248', - 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', - 'One of the only ways to improve Alexa is to have human beings check it for errors', - 'theverge.com'], + 'LinkedIn', + 'This link will take you to a page that’s not on LinkedIn', + 'lnkd.in'], ['Thom_Wolf/status/1122466524860702729', 'facebookresearch/fairseq', From e1838e093335fab02f36649bba7b65aa4420993f Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 05:09:21 +0000 Subject: [PATCH 20/31] Move CI workflow to buildjet --- .github/workflows/run-tests.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 76af329..ee28e33 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -10,24 +10,26 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: buildjet-2vcpu-ubuntu-2204 strategy: matrix: nim: - "1.6.10" - "1.6.x" - "2.0.x" + - "devel" steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache nimble id: cache-nimble - uses: actions/cache@v3 + uses: buildjet/cache@v3 with: path: ~/.nimble - key: nimble-${{ hashFiles('*.nimble') }} - restore-keys: "nimble-" + key: ${{ matrix.nim }}-nimble-${{ hashFiles('*.nimble') }} + restore-keys: | + ${{ matrix.nim }}-nimble- - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -51,4 +53,4 @@ jobs: - name: Run tests run: | ./nitter & - pytest -n4 tests + pytest -n8 tests From 209f453b7998d0ac43950618d61f87efbf8c2b26 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 05:09:44 +0000 Subject: [PATCH 21/31] Purge expired accounts after parsing --- src/auth.nim | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index 8be435c..8c901e2 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,5 +1,5 @@ #SPDX-License-Identifier: AGPL-3.0-only -import std/[asyncdispatch, times, json, random, strutils, tables, packedsets, os] +import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] import types import experimental/parser/guestaccount @@ -30,6 +30,16 @@ var template log(str: varargs[string, `$`]) = if enableLogging: echo "[accounts] ", str.join("") +proc snowflakeToEpoch(flake: int64): int64 = + int64(((flake shr 22) + 1288834974657) div 1000) + +proc hasExpired(account: GuestAccount): bool = + let + created = snowflakeToEpoch(account.id) + now = epochTime().int64 + daysOld = int(now - created) div (24 * 60 * 60) + return daysOld > 30 + proc getAccountPoolHealth*(): JsonNode = let now = epochTime().int @@ -42,9 +52,7 @@ proc getAccountPoolHealth*(): JsonNode = average = 0'i64 for account in accountPool: - # Twitter snowflake conversion - let created = int64(((account.id shr 22) + 1288834974657) div 1000) - + let created = snowflakeToEpoch(account.id) if created > newest: newest = created if created < oldest: @@ -188,3 +196,10 @@ proc initAccountPool*(cfg: Config; path: string) = else: echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." quit 1 + + let accountsPrePurge = accountPool.len + accountPool.keepItIf(not it.hasExpired) + + log "Successfully added ", accountPool.len, " valid accounts." + if accountsPrePurge > accountPool.len: + log "Purged ", accountsPrePurge - accountPool.len, " expired accounts." From d17583286a11586c6ff5cffc43bc997e525a578e Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 1 Nov 2023 05:44:08 +0000 Subject: [PATCH 22/31] Don't requests made before reset --- src/auth.nim | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index 8c901e2..fed7df3 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -64,12 +64,16 @@ proc getAccountPoolHealth*(): JsonNode = apiStatus = account.apis[api] reqs = apiMaxReqs[api] - apiStatus.remaining - reqsPerApi.mgetOrPut($api, 0).inc reqs - totalReqs.inc reqs - if apiStatus.limited: limited.incl account.id + # no requests made with this account and endpoint since the limit reset + if apiStatus.reset < now: + continue + + reqsPerApi.mgetOrPut($api, 0).inc reqs + totalReqs.inc reqs + if accountPool.len > 0: average = average div accountPool.len else: From e0d9dd0f9c8175fe9f5bf5aa86c0f56ccbc970a9 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Nov 2023 02:56:32 +0000 Subject: [PATCH 23/31] Fix #670 --- src/auth.nim | 4 ++-- src/sass/profile/card.scss | 2 +- src/types.nim | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index fed7df3..b288c50 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -37,7 +37,7 @@ proc hasExpired(account: GuestAccount): bool = let created = snowflakeToEpoch(account.id) now = epochTime().int64 - daysOld = int(now - created) div (24 * 60 * 60) + daysOld = int(now - created) div dayInSeconds return daysOld > 30 proc getAccountPoolHealth*(): JsonNode = @@ -45,7 +45,7 @@ proc getAccountPoolHealth*(): JsonNode = var totalReqs = 0 - limited: PackedSet[BiggestInt] + limited: PackedSet[int64] reqsPerApi: Table[string, int] oldest = now.int64 newest = 0'i64 diff --git a/src/sass/profile/card.scss b/src/sass/profile/card.scss index 85878e4..46a9679 100644 --- a/src/sass/profile/card.scss +++ b/src/sass/profile/card.scss @@ -115,7 +115,7 @@ } .profile-card-tabs-name { - @include breakable; + flex-shrink: 100; } .profile-card-avatar { diff --git a/src/types.nim b/src/types.nim index 3b0d55c..9ddf283 100644 --- a/src/types.nim +++ b/src/types.nim @@ -36,7 +36,7 @@ type limitedAt*: int GuestAccount* = ref object - id*: BiggestInt + id*: int64 oauthToken*: string oauthSecret*: string pending*: int @@ -164,7 +164,7 @@ type newsletterPublication = "newsletter_publication" hidden unknown - + Card* = object kind*: CardKind url*: string From 5e188647fc5ddcc38084127f1db32f17f07fe727 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 8 Nov 2023 14:53:35 +0000 Subject: [PATCH 24/31] Bump Nim in the ARM64 Dockerfile, add nitter user --- Dockerfile.arm64 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index fbad812..c82be8a 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1,7 +1,7 @@ FROM alpine:3.18 as nim LABEL maintainer="setenforce@protonmail.com" -RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.14-r0" nimble pcre +RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.16-r0" "nimble=0.13.1-r3" WORKDIR /src/nitter @@ -15,9 +15,11 @@ RUN nimble build -d:danger -d:lto -d:strip \ FROM alpine:3.18 WORKDIR /src/ -RUN apk --no-cache add ca-certificates pcre openssl1.1-compat +RUN apk --no-cache add pcre ca-certificates openssl1.1-compat COPY --from=nim /src/nitter/nitter ./ COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf COPY --from=nim /src/nitter/public ./public EXPOSE 8080 +RUN adduser -h /src/ -D -s /bin/sh nitter +USER nitter CMD ./nitter From eaedd2aee7be6bc3dd2dceee09dc93052d0046f4 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 8 Nov 2023 16:38:43 +0000 Subject: [PATCH 25/31] Fix ARM64 Dockerfile versions --- Dockerfile.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index c82be8a..70024b2 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1,7 +1,7 @@ FROM alpine:3.18 as nim LABEL maintainer="setenforce@protonmail.com" -RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.16-r0" "nimble=0.13.1-r3" +RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.14-r0" "nimble=0.13.1-r2" WORKDIR /src/nitter From c2819dab441b8ad8220b03dd0fe79f5a5d51b841 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 15 Nov 2023 10:40:21 +0000 Subject: [PATCH 26/31] Fix #1106 Closes #831 --- src/parserutils.nim | 14 ++++++++++++-- tests/test_quote.py | 2 +- tests/test_tweet.py | 11 ++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/parserutils.nim b/src/parserutils.nim index 7cf696e..6b8263f 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -1,9 +1,17 @@ # SPDX-License-Identifier: AGPL-3.0-only -import std/[strutils, times, macros, htmlgen, options, algorithm, re] +import std/[times, macros, htmlgen, options, algorithm, re] +import std/strutils except escape import std/unicode except strip +from xmltree import escape import packedjson import types, utils, formatters +const + unicodeOpen = "\uFFFA" + unicodeClose = "\uFFFB" + xmlOpen = escape("<") + xmlClose = escape(">") + let unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})" unReplace = "$1@$2" @@ -304,7 +312,9 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let entities = ? js{"entity_set"} - text = js{"text"}.getStr + text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose)) textSlice = 0..text.runeLen tweet.expandTextEntities(entities, text, textSlice) + + tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose)) diff --git a/tests/test_quote.py b/tests/test_quote.py index 1b458ea..4921c21 100644 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -9,7 +9,7 @@ text = [ What are we doing wrong? reuters.com/article/us-norwa…"""], ['nim_lang/status/1491461266849808397#m', - 'Nim language', '@nim_lang', + 'Nim', '@nim_lang', """What's better than Nim 1.6.0? Nim 1.6.2 :) diff --git a/tests/test_tweet.py b/tests/test_tweet.py index 7a3c4ed..ac89782 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -35,7 +35,16 @@ multiline = [ CALM AND CLICHÉ - ON"""] + ON"""], + [1718660434457239868, 'WebDesignMuseum', + """ +Happy 32nd Birthday HTML tags! + +On October 29, 1991, the internet pioneer, Tim Berners-Lee, published a document entitled HTML Tags. + +The document contained a description of the first 18 HTML tags: , <nextid>, <a>, <isindex>, <plaintext>, <listing>, <p>, <h1>…<h6>, <address>, <hp1>, <hp2>…, <dl>, <dt>, <dd>, <ul>, <li>,<menu> and <dir>. The design of the first version of HTML language was influenced by the SGML universal markup language. + +#WebDesignHistory"""] ] link = [ From 06ab1ea2e7341a239447e0ca7d1e9c6246b896c6 Mon Sep 17 00:00:00 2001 From: Zed <zedeus@pm.me> Date: Wed, 15 Nov 2023 11:11:56 +0000 Subject: [PATCH 27/31] Enable disabled tests --- tests/test_tweet.py | 58 +++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/tests/test_tweet.py b/tests/test_tweet.py index ac89782..839e6c5 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -1,4 +1,4 @@ -from base import BaseTestCase, Tweet, get_timeline_tweet +from base import BaseTestCase, Tweet, Conversation, get_timeline_tweet from parameterized import parameterized # image = tweet + 'div.attachments.media-body > div > div > a > div > img' @@ -83,22 +83,18 @@ retweet = [ [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr'] ] -# reply = [ -# ['mobile_test/with_replies', 15] -# ] - class TweetTest(BaseTestCase): - # @parameterized.expand(timeline) - # def test_timeline(self, index, fullname, username, date, tid, text): - # self.open_nitter(username) - # tweet = get_timeline_tweet(index) - # self.assert_exact_text(fullname, tweet.fullname) - # self.assert_exact_text('@' + username, tweet.username) - # self.assert_exact_text(date, tweet.date) - # self.assert_text(text, tweet.text) - # permalink = self.find_element(tweet.date + ' a') - # self.assertIn(tid, permalink.get_attribute('href')) + @parameterized.expand(timeline) + def test_timeline(self, index, fullname, username, date, tid, text): + self.open_nitter(username) + tweet = get_timeline_tweet(index) + self.assert_exact_text(fullname, tweet.fullname) + self.assert_exact_text('@' + username, tweet.username) + self.assert_exact_text(date, tweet.date) + self.assert_text(text, tweet.text) + permalink = self.find_element(tweet.date + ' a') + self.assertIn(tid, permalink.get_attribute('href')) @parameterized.expand(status) def test_status(self, tid, fullname, username, date, text): @@ -112,18 +108,18 @@ class TweetTest(BaseTestCase): @parameterized.expand(multiline) def test_multiline_formatting(self, tid, username, text): self.open_nitter(f'{username}/status/{tid}') - self.assert_text(text.strip('\n'), '.main-tweet') + self.assert_text(text.strip('\n'), Conversation.main) @parameterized.expand(emoji) def test_emoji(self, tweet, text): self.open_nitter(tweet) - self.assert_text(text, '.main-tweet') + self.assert_text(text, Conversation.main) @parameterized.expand(link) def test_link(self, tweet, links): self.open_nitter(tweet) for link in links: - self.assert_text(link, '.main-tweet') + self.assert_text(link, Conversation.main) @parameterized.expand(username) def test_username(self, tweet, usernames): @@ -132,22 +128,22 @@ class TweetTest(BaseTestCase): link = self.find_link_text(f'@{un}') self.assertIn(f'/{un}', link.get_property('href')) - # @parameterized.expand(retweet) - # def test_retweet(self, index, url, retweet_by, fullname, username, text): - # self.open_nitter(url) - # tweet = get_timeline_tweet(index) - # self.assert_text(f'{retweet_by} retweeted', tweet.retweet) - # self.assert_text(text, tweet.text) - # self.assert_exact_text(fullname, tweet.fullname) - # self.assert_exact_text(username, tweet.username) + @parameterized.expand(retweet) + def test_retweet(self, index, url, retweet_by, fullname, username, text): + self.open_nitter(url) + tweet = get_timeline_tweet(index) + self.assert_text(f'{retweet_by} retweeted', tweet.retweet) + self.assert_text(text, tweet.text) + self.assert_exact_text(fullname, tweet.fullname) + self.assert_exact_text(username, tweet.username) @parameterized.expand(invalid) def test_invalid_id(self, tweet): self.open_nitter(tweet) self.assert_text('Tweet not found', '.error-panel') - # @parameterized.expand(reply) - # def test_thread(self, tweet, num): - # self.open_nitter(tweet) - # thread = self.find_element(f'.timeline > div:nth-child({num})') - # self.assertIn(thread.get_attribute('class'), 'thread-line') + #@parameterized.expand(reply) + #def test_thread(self, tweet, num): + #self.open_nitter(tweet) + #thread = self.find_element(f'.timeline > div:nth-child({num})') + #self.assertIn(thread.get_attribute('class'), 'thread-line') From 4dac9f0798b885130b01f9a5a5535c729d351467 Mon Sep 17 00:00:00 2001 From: Zed <zedeus@pm.me> Date: Sat, 25 Nov 2023 05:31:15 +0000 Subject: [PATCH 28/31] Add simple job_details card support --- src/experimental/parser/graphql.nim | 4 ++-- src/experimental/parser/unifiedcard.nim | 11 +++++++++ src/experimental/parser/user.nim | 7 +++++- src/experimental/types/graphuser.nim | 4 ++-- src/experimental/types/timeline.nim | 4 ++-- src/experimental/types/unifiedcard.nim | 32 ++++++++++++++++++++----- src/parser.nim | 4 ++-- src/parserutils.nim | 9 +++---- src/types.nim | 1 + src/utils.nim | 5 ++-- 10 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index b9da7c4..0e9a678 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -12,7 +12,7 @@ proc parseGraphUser*(json: string): User = if raw.data.userResult.result.unavailableReason.get("") == "Suspended": return User(suspended: true) - result = toUser raw.data.userResult.result.legacy + result = raw.data.userResult.result.legacy result.id = raw.data.userResult.result.restId result.verified = result.verified or raw.data.userResult.result.isBlueVerified @@ -30,7 +30,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] = of TimelineTimelineItem: let userResult = entry.content.itemContent.userResults.result if userResult.restId.len > 0: - result.content.add toUser userResult.legacy + result.content.add userResult.legacy of TimelineTimelineCursor: if entry.content.cursorType == "Bottom": result.bottom = entry.content.value diff --git a/src/experimental/parser/unifiedcard.nim b/src/experimental/parser/unifiedcard.nim index c9af437..4a50e48 100644 --- a/src/experimental/parser/unifiedcard.nim +++ b/src/experimental/parser/unifiedcard.nim @@ -1,5 +1,6 @@ import std/[options, tables, strutils, strformat, sugar] import jsony +import user import ../types/unifiedcard from ../../types import Card, CardKind, Video from ../../utils import twimg, https @@ -27,6 +28,14 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) result.text = data.topicDetail.title result.dest = "Topic" +proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) = + data.destination.parseDestination(card, result) + + result.kind = jobDetails + result.title = data.title + result.text = data.shortDescriptionText + result.dest = &"@{data.profileUser.username} · {data.location}" + proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) = let app = card.appStoreData[data.appId][0] @@ -84,6 +93,8 @@ proc parseUnifiedCard*(json: string): Card = component.parseMedia(card, result) of buttonGroup: discard + of jobDetails: + component.data.parseJobDetails(card, result) of ComponentType.hidden: result.kind = CardKind.hidden of ComponentType.unknown: diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index 5962a87..78f596e 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -68,6 +68,11 @@ proc toUser*(raw: RawUser): User = result.expandUserEntities(raw) +proc parseHook*(s: string; i: var int; v: var User) = + var u: RawUser + parseHook(s, i, u) + v = toUser u + proc parseUser*(json: string; username=""): User = handleErrors: case error.code @@ -75,7 +80,7 @@ proc parseUser*(json: string; username=""): User = of userNotFound: return else: echo "[error - parseUser]: ", error - result = toUser json.fromJson(RawUser) + result = json.fromJson(User) proc parseUsers*(json: string; after=""): Result[User] = result = Result[User](beginning: after.len == 0) diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index c30eed9..08100f9 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -1,5 +1,5 @@ import options -import user +from ../../types import User type GraphUser* = object @@ -9,7 +9,7 @@ type result*: UserResult UserResult = object - legacy*: RawUser + legacy*: User restId*: string isBlueVerified*: bool unavailableReason*: Option[string] diff --git a/src/experimental/types/timeline.nim b/src/experimental/types/timeline.nim index 28239ad..5ce6d9f 100644 --- a/src/experimental/types/timeline.nim +++ b/src/experimental/types/timeline.nim @@ -1,5 +1,5 @@ import std/tables -import user +from ../../types import User type Search* = object @@ -7,7 +7,7 @@ type timeline*: Timeline GlobalObjects = object - users*: Table[string, RawUser] + users*: Table[string, User] Timeline = object instructions*: seq[Instructions] diff --git a/src/experimental/types/unifiedcard.nim b/src/experimental/types/unifiedcard.nim index 6e83cad..e540a64 100644 --- a/src/experimental/types/unifiedcard.nim +++ b/src/experimental/types/unifiedcard.nim @@ -1,7 +1,10 @@ -import options, tables -from ../../types import VideoType, VideoVariant +import std/[options, tables, times] +import jsony +from ../../types import VideoType, VideoVariant, User type + Text* = distinct string + UnifiedCard* = object componentObjects*: Table[string, Component] destinationObjects*: Table[string, Destination] @@ -13,6 +16,7 @@ type media swipeableMedia buttonGroup + jobDetails appStoreDetails twitterListDetails communityDetails @@ -29,12 +33,15 @@ type appId*: string mediaId*: string destination*: string + location*: string title*: Text subtitle*: Text name*: Text memberCount*: int mediaList*: seq[MediaItem] topicDetail*: tuple[title: Text] + profileUser*: User + shortDescriptionText*: string MediaItem* = object id*: string @@ -69,12 +76,9 @@ type title*: Text category*: Text - Text = object - content: string - TypeField = Component | Destination | MediaEntity | AppStoreData -converter fromText*(text: Text): string = text.content +converter fromText*(text: Text): string = string(text) proc renameHook*(v: var TypeField; fieldName: var string) = if fieldName == "type": @@ -86,6 +90,7 @@ proc enumHook*(s: string; v: var ComponentType) = of "media": media of "swipeable_media": swipeableMedia of "button_group": buttonGroup + of "job_details": jobDetails of "app_store_details": appStoreDetails of "twitter_list_details": twitterListDetails of "community_details": communityDetails @@ -106,3 +111,18 @@ proc enumHook*(s: string; v: var MediaType) = of "photo": photo of "model3d": model3d else: echo "ERROR: Unknown enum value (MediaType): ", s; photo + +proc parseHook*(s: string; i: var int; v: var DateTime) = + var str: string + parseHook(s, i, str) + v = parse(str, "yyyy-MM-dd hh:mm:ss") + +proc parseHook*(s: string; i: var int; v: var Text) = + if s[i] == '"': + var str: string + parseHook(s, i, str) + v = Text(str) + else: + var t: tuple[content: string] + parseHook(s, i, t) + v = Text(t.content) diff --git a/src/parser.nim b/src/parser.nim index fcee13f..d7ba613 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -219,8 +219,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = ) ) - result.expandTweetEntities(js) - # fix for pinned threads if result.hasThread and result.threadId == 0: result.threadId = js{"self_thread", "id_str"}.getId @@ -254,6 +252,8 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = else: result.card = some parseCard(jsCard, js{"entities", "urls"}) + result.expandTweetEntities(js) + with jsMedia, js{"extended_entities", "media"}: for m in jsMedia: case m{"type"}.getStr diff --git a/src/parserutils.nim b/src/parserutils.nim index 6b8263f..00ea6f4 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -246,7 +246,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) = .replacef(htRegex, htReplace) proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int]; - replyTo=""; hasQuote=false) = + replyTo=""; hasRedundantLink=false) = let hasCard = tweet.card.isSome var replacements = newSeq[ReplaceSlice]() @@ -257,7 +257,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic if urlStr.len == 0 or urlStr notin text: continue - replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote) + replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink) if hasCard and u{"url"}.getStr == get(tweet.card).url: get(tweet.card).url = u{"expanded_url"}.getStr @@ -297,9 +297,10 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = let entities = ? js{"entities"} - hasQuote = js{"is_quote_status"}.getBool textRange = js{"display_text_range"} textSlice = textRange{0}.getInt .. textRange{1}.getInt + hasQuote = js{"is_quote_status"}.getBool + hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails var replyTo = "" if tweet.replyId != 0: @@ -307,7 +308,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = replyTo = reply.getStr tweet.reply.add replyTo - tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote) + tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard) proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let diff --git a/src/types.nim b/src/types.nim index 9ddf283..bc791b1 100644 --- a/src/types.nim +++ b/src/types.nim @@ -162,6 +162,7 @@ type imageDirectMessage = "image_direct_message" audiospace = "audiospace" newsletterPublication = "newsletter_publication" + jobDetails = "job_details" hidden unknown diff --git a/src/utils.nim b/src/utils.nim index 9002bbf..c96a6dd 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -16,7 +16,8 @@ const "twimg.com", "abs.twimg.com", "pbs.twimg.com", - "video.twimg.com" + "video.twimg.com", + "x.com" ] proc setHmacKey*(key: string) = @@ -57,4 +58,4 @@ proc isTwitterUrl*(uri: Uri): bool = uri.hostname in twitterDomains proc isTwitterUrl*(url: string): bool = - parseUri(url).hostname in twitterDomains + isTwitterUrl(parseUri(url)) From d6be08d093bcde38286268b789395cf7a99e67ed Mon Sep 17 00:00:00 2001 From: Zed <zedeus@pm.me> Date: Sat, 25 Nov 2023 05:53:13 +0000 Subject: [PATCH 29/31] Fix jobDetails error on old Nim versions --- src/experimental/parser/unifiedcard.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/experimental/parser/unifiedcard.nim b/src/experimental/parser/unifiedcard.nim index 4a50e48..1f7b825 100644 --- a/src/experimental/parser/unifiedcard.nim +++ b/src/experimental/parser/unifiedcard.nim @@ -31,7 +31,7 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) = data.destination.parseDestination(card, result) - result.kind = jobDetails + result.kind = CardKind.jobDetails result.title = data.title result.text = data.shortDescriptionText result.dest = &"@{data.profileUser.username} · {data.location}" @@ -93,7 +93,7 @@ proc parseUnifiedCard*(json: string): Card = component.parseMedia(card, result) of buttonGroup: discard - of jobDetails: + of ComponentType.jobDetails: component.data.parseJobDetails(card, result) of ComponentType.hidden: result.kind = CardKind.hidden From f8254c2f0f3bfacb754d7ad69be9b55258c8337c Mon Sep 17 00:00:00 2001 From: Zed <zedeus@pm.me> Date: Sat, 25 Nov 2023 10:06:12 +0000 Subject: [PATCH 30/31] Add support for business and gov verification Also improve icon rendering on Firefox --- src/consts.nim | 10 ++++------ src/experimental/parser/graphql.nim | 5 +++-- src/experimental/parser/unifiedcard.nim | 3 +-- src/experimental/parser/user.nim | 2 +- src/experimental/types/user.nim | 4 ++-- src/parser.nim | 6 +++--- src/redis_cache.nim | 1 + src/sass/include/_variables.scss | 2 ++ src/sass/index.scss | 21 ++++++++++++++++++--- src/sass/search.scss | 2 ++ src/types.nim | 12 ++++++++---- src/views/general.nim | 2 +- src/views/renderutils.nim | 17 ++++++++++++----- src/views/tweet.nim | 3 +-- 14 files changed, 59 insertions(+), 31 deletions(-) diff --git a/src/consts.nim b/src/consts.nim index d3a3d80..e1c35e6 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -29,12 +29,10 @@ const "include_cards": "1", "include_entities": "1", "include_profile_interstitial_type": "0", - "include_quote_count": "1", - "include_reply_count": "1", - "include_user_entities": "1", - "include_ext_reply_count": "1", - "include_ext_is_blue_verified": "1", - # "include_ext_verified_type": "1", + "include_quote_count": "0", + "include_reply_count": "0", + "include_user_entities": "0", + "include_ext_reply_count": "0", "include_ext_media_color": "0", "cards_platform": "Web-13", "tweet_mode": "extended", diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index 0e9a678..c7f115f 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -1,7 +1,7 @@ import options import jsony import user, ../types/[graphuser, graphlistmembers] -from ../../types import User, Result, Query, QueryKind +from ../../types import User, VerifiedType, Result, Query, QueryKind proc parseGraphUser*(json: string): User = if json.len == 0 or json[0] != '{': @@ -14,7 +14,8 @@ proc parseGraphUser*(json: string): User = result = raw.data.userResult.result.legacy result.id = raw.data.userResult.result.restId - result.verified = result.verified or raw.data.userResult.result.isBlueVerified + if result.verifiedType == none and raw.data.userResult.result.isBlueVerified: + result.verifiedType = blue proc parseGraphListMembers*(json, cursor: string): Result[User] = result = Result[User]( diff --git a/src/experimental/parser/unifiedcard.nim b/src/experimental/parser/unifiedcard.nim index 1f7b825..a112974 100644 --- a/src/experimental/parser/unifiedcard.nim +++ b/src/experimental/parser/unifiedcard.nim @@ -1,7 +1,6 @@ import std/[options, tables, strutils, strformat, sugar] import jsony -import user -import ../types/unifiedcard +import user, ../types/unifiedcard from ../../types import Card, CardKind, Video from ../../utils import twimg, https diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index 78f596e..07e0477 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -56,7 +56,7 @@ proc toUser*(raw: RawUser): User = tweets: raw.statusesCount, likes: raw.favouritesCount, media: raw.mediaCount, - verified: raw.verified or raw.extIsBlueVerified, + verifiedType: raw.verifiedType, protected: raw.protected, joinDate: parseTwitterDate(raw.createdAt), banner: getBanner(raw), diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim index 39331a0..7dc0194 100644 --- a/src/experimental/types/user.nim +++ b/src/experimental/types/user.nim @@ -1,5 +1,6 @@ import options import common +from ../../types import VerifiedType type RawUser* = object @@ -15,8 +16,7 @@ type favouritesCount*: int statusesCount*: int mediaCount*: int - verified*: bool - extIsBlueVerified*: bool + verifiedType*: VerifiedType protected*: bool profileLinkColor*: string profileBannerUrl*: string diff --git a/src/parser.nim b/src/parser.nim index d7ba613..a7bf89d 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -21,7 +21,7 @@ proc parseUser(js: JsonNode; id=""): User = tweets: js{"statuses_count"}.getInt, likes: js{"favourites_count"}.getInt, media: js{"media_count"}.getInt, - verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool, + verifiedType: parseEnum[VerifiedType](js{"verified_type"}.getStr("None")), protected: js{"protected"}.getBool, joinDate: js{"created_at"}.getTime ) @@ -34,8 +34,8 @@ proc parseGraphUser(js: JsonNode): User = user = ? js{"user_results", "result"} result = parseUser(user{"legacy"}) - if "is_blue_verified" in user: - result.verified = user{"is_blue_verified"}.getBool() + if result.verifiedType == none and user{"is_blue_verified"}.getBool(false): + result.verifiedType = blue proc parseGraphList*(js: JsonNode): List = if js.isNull: return diff --git a/src/redis_cache.nim b/src/redis_cache.nim index a8b5ff8..1d77cca 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -52,6 +52,7 @@ proc initRedisPool*(cfg: Config) {.async.} = await migrate("profileDates", "p:*") await migrate("profileStats", "p:*") await migrate("userType", "p:*") + await migrate("verifiedType", "p:*") pool.withAcquire(r): # optimize memory usage for user ID buckets diff --git a/src/sass/include/_variables.scss b/src/sass/include/_variables.scss index 0f81235..0c95ff6 100644 --- a/src/sass/include/_variables.scss +++ b/src/sass/include/_variables.scss @@ -28,6 +28,8 @@ $more_replies_dots: #AD433B; $error_red: #420A05; $verified_blue: #1DA1F2; +$verified_business: #FAC82B; +$verified_government: #C1B6A4; $icon_text: $fg_color; $tab: $fg_color; diff --git a/src/sass/index.scss b/src/sass/index.scss index 9e2e347..6cab48e 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -39,6 +39,8 @@ body { --error_red: #{$error_red}; --verified_blue: #{$verified_blue}; + --verified_business: #{$verified_business}; + --verified_government: #{$verified_government}; --icon_text: #{$icon_text}; --tab: #{$fg_color}; @@ -141,17 +143,30 @@ ul { .verified-icon { color: var(--icon_text); - background-color: var(--verified_blue); border-radius: 50%; flex-shrink: 0; margin: 2px 0 3px 3px; - padding-top: 2px; - height: 12px; + padding-top: 3px; + height: 11px; width: 14px; font-size: 8px; display: inline-block; text-align: center; vertical-align: middle; + + &.blue { + background-color: var(--verified_blue); + } + + &.business { + color: var(--bg_panel); + background-color: var(--verified_business); + } + + &.government { + color: var(--bg_panel); + background-color: var(--verified_government); + } } @media(max-width: 600px) { diff --git a/src/sass/search.scss b/src/sass/search.scss index 0311fb0..f70f7ea 100644 --- a/src/sass/search.scss +++ b/src/sass/search.scss @@ -14,6 +14,8 @@ button { margin: 0 2px 0 0; height: 23px; + display: flex; + align-items: center; } .pref-input { diff --git a/src/types.nim b/src/types.nim index bc791b1..ddbebdf 100644 --- a/src/types.nim +++ b/src/types.nim @@ -10,9 +10,7 @@ type BadClientError* = object of CatchableError TimelineKind* {.pure.} = enum - tweets - replies - media + tweets, replies, media Api* {.pure.} = enum tweetDetail @@ -63,6 +61,12 @@ type tweetUnavailable = 421 tweetCensored = 422 + VerifiedType* = enum + none = "None" + blue = "Blue" + business = "Business" + government = "Government" + User* = object id*: string username*: string @@ -78,7 +82,7 @@ type tweets*: int likes*: int media*: int - verified*: bool + verifiedType*: VerifiedType protected*: bool suspended*: bool joinDate*: DateTime diff --git a/src/views/general.nim b/src/views/general.nim index 5e96d02..87d30f2 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=18") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=19") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2") if theme.len > 0: diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 9dffdcb..451ddfb 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -23,6 +23,13 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = if text.len > 0: text " " & text +template verifiedIcon*(user: User): untyped {.dirty.} = + if user.verifiedType != none: + let lower = ($user.verifiedType).toLowerAscii() + icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account") + else: + text "" + proc linkUser*(user: User, class=""): VNode = let isName = "username" notin class @@ -32,11 +39,11 @@ proc linkUser*(user: User, class=""): VNode = buildHtml(a(href=href, class=class, title=nameText)): text nameText - if isName and user.verified: - icon "ok", class="verified-icon", title="Verified account" - if isName and user.protected: - text " " - icon "lock", title="Protected account" + if isName: + verifiedIcon(user) + if user.protected: + text " " + icon "lock", title="Protected account" proc linkText*(text: string; class=""): VNode = let url = if "http" notin text: https & text else: text diff --git a/src/views/tweet.nim b/src/views/tweet.nim index f47ae9a..2fe4ac9 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -200,8 +200,7 @@ proc renderAttribution(user: User; prefs: Prefs): VNode = buildHtml(a(class="attribution", href=("/" & user.username))): renderMiniAvatar(user, prefs) strong: text user.fullname - if user.verified: - icon "ok", class="verified-icon", title="Verified account" + verifiedIcon(user) proc renderMediaTags(tags: seq[User]): VNode = buildHtml(tdiv(class="media-tag-block")): From a9740fec8b2d9d4469322cc66d3f3547e0b6ccdb Mon Sep 17 00:00:00 2001 From: Zed <zedeus@pm.me> Date: Sat, 25 Nov 2023 10:11:57 +0000 Subject: [PATCH 31/31] Fix compilation with old Nim again --- src/experimental/parser/graphql.nim | 2 +- src/parser.nim | 2 +- src/views/renderutils.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index c7f115f..69837ab 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -14,7 +14,7 @@ proc parseGraphUser*(json: string): User = result = raw.data.userResult.result.legacy result.id = raw.data.userResult.result.restId - if result.verifiedType == none and raw.data.userResult.result.isBlueVerified: + if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified: result.verifiedType = blue proc parseGraphListMembers*(json, cursor: string): Result[User] = diff --git a/src/parser.nim b/src/parser.nim index a7bf89d..f2547e4 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -34,7 +34,7 @@ proc parseGraphUser(js: JsonNode): User = user = ? js{"user_results", "result"} result = parseUser(user{"legacy"}) - if result.verifiedType == none and user{"is_blue_verified"}.getBool(false): + if result.verifiedType == VerifiedType.none and user{"is_blue_verified"}.getBool(false): result.verifiedType = blue proc parseGraphList*(js: JsonNode): List = diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 451ddfb..f298fad 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -24,7 +24,7 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = text " " & text template verifiedIcon*(user: User): untyped {.dirty.} = - if user.verifiedType != none: + if user.verifiedType != VerifiedType.none: let lower = ($user.verifiedType).toLowerAscii() icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account") else: