From 4250245263b77bd2b0ecd9a31c0e4000d8e2e6f7 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 2 Sep 2023 07:28:56 +0200 Subject: [PATCH 1/3] Shorten media proxy error log --- src/routes/media.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/media.nim b/src/routes/media.nim index d335c97..eacd1f8 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -37,7 +37,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = try: let res = await client.get(url) if res.status != "200 OK": - echo "[media] Proxying media failed, status: $1, url: $2, body: $3" % [res.status, url, await res.body] + echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url] return Http404 let hashed = $hash(url) @@ -66,7 +66,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = await request.client.send(data) data.setLen 0 except HttpRequestError, ProtocolError, OSError: - echo "[media] Proxying media exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url] + echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url] result = Http404 finally: client.close() From fcd74e8048362fcf8284871ee067099e8de28a89 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 2 Sep 2023 08:15:58 +0200 Subject: [PATCH 2/3] Retry rate limited requests with different account --- src/apiutils.nim | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index 0b1db26..9ac101e 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -120,28 +120,38 @@ template fetchImpl(result, fetchBody) {.dirty.} = except OSError as e: raise e except Exception as e: - echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", account.id, ", url: ", url + let id = if account.isNil: "null" else: account.id + echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url raise rateLimitError() finally: release(account) -proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = - var body: string - fetchImpl body: - if body.startsWith('{') or body.startsWith('['): - result = parseJson(body) - else: - echo resp.status, ": ", body, " --- url: ", url - result = newJNull() +template retry(bod) = + try: + bod + except RateLimitError: + echo "[accounts] Rate limited, retrying ", api, " request..." + bod - let error = result.getError - if error in {expiredToken, badToken}: - echo "fetchBody error: ", error - invalidate(account) - raise rateLimitError() +proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = + retry: + var body: string + fetchImpl body: + if body.startsWith('{') or body.startsWith('['): + result = parseJson(body) + else: + echo resp.status, ": ", body, " --- url: ", url + result = newJNull() + + let error = result.getError + if error in {expiredToken, badToken}: + echo "fetchBody error: ", error + invalidate(account) + raise rateLimitError() proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = - fetchImpl result: - if not (result.startsWith('{') or result.startsWith('[')): - echo resp.status, ": ", result, " --- url: ", url - result.setLen(0) + retry: + fetchImpl result: + if not (result.startsWith('{') or result.startsWith('[')): + echo resp.status, ": ", result, " --- url: ", url + result.setLen(0) From 14f9a092d832c0aaf7eb4900ab36a76c6653c364 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 14 Sep 2023 23:35:41 +0000 Subject: [PATCH 3/3] Fix crash on missing quote tweet data crash --- src/parser.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/parser.nim b/src/parser.nim index 914c038..776f176 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -324,6 +324,9 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet = of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}, isLegacy) + if not js.hasKey("legacy"): + return Tweet() + var jsCard = copy(js{if isLegacy: "card" else: "tweet_card", "legacy"}) if jsCard.kind != JNull: var values = newJObject()