From 3d8858f0d86d09ce815bc78db417290557f23908 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 20 Aug 2023 11:56:42 +0200 Subject: [PATCH] Track rate limits, reset after 24 hours --- src/apiutils.nim | 12 +++++++++++- src/tokens.nim | 26 ++++++++++++++++++++------ src/types.nim | 2 ++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index d1ecfa3..54e6777 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import httpclient, asyncdispatch, options, strutils, uri, times, math +import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import jsony, packedjson, zippy, oauth1 import types, tokens, consts, parserutils, http_pool import experimental/types/common @@ -129,6 +129,16 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = release(account, invalid=true) raise rateLimitError() + if body.startsWith("{\"errors"): + let errors = body.fromJson(Errors) + if errors in {invalidToken, badToken}: + echo "fetch error: ", errors + release(account, invalid=true) + raise rateLimitError() + elif errors in {rateLimited}: + account.apis[api].limited = true + echo "rate limited, api: ", $api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id + proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = fetchImpl result: if not (result.startsWith('{') or result.startsWith('[')): diff --git a/src/tokens.nim b/src/tokens.nim index 401dc05..45aa895 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -3,7 +3,9 @@ import asyncdispatch, times, json, random, strutils, tables import types # max requests at a time per account to avoid race conditions -const maxConcurrentReqs = 5 +const + maxConcurrentReqs = 5 + dayInSeconds = 24 * 60 * 60 var accountPool: seq[GuestAccount] @@ -19,7 +21,7 @@ proc getPoolJson*(): JsonNode = totalPending = 0 reqsPerApi: Table[string, int] - let now = epochTime() + let now = epochTime().int for account in accountPool: totalPending.inc(account.pending) @@ -29,10 +31,17 @@ proc getPoolJson*(): JsonNode = } for api in account.apis.keys: - if (now.int - account.apis[api].reset) / 60 > 15: - continue + let obj = %*{} + if account.apis[api].limited: + obj["limited"] = %true - list[account.id]["apis"][$api] = %account.apis[api].remaining + if account.apis[api].reset > now.int: + obj["remaining"] = %account.apis[api].remaining + + list[account.id]["apis"][$api] = obj + + if "remaining" notin obj: + continue let maxReqs = @@ -65,7 +74,12 @@ proc isLimited(account: GuestAccount; api: Api): bool = if api in account.apis: let limit = account.apis[api] - return (limit.remaining <= 10 and limit.reset > epochTime().int) + + if limit.limited and (epochTime().int - limit.limitedAt) > dayInSeconds: + account.apis[api].limited = false + echo "account limit reset, api: ", api, ", id: ", account.id + + return limit.limited or (limit.remaining <= 10 and limit.reset > epochTime().int) else: return false diff --git a/src/types.nim b/src/types.nim index 2a553dd..33d0cda 100644 --- a/src/types.nim +++ b/src/types.nim @@ -32,6 +32,8 @@ type RateLimit* = object remaining*: int reset*: int + limited*: bool + limitedAt*: int GuestAccount* = ref object id*: string