Optional base64 support for proxy urls

This commit is contained in:
Zed 2020-06-09 15:04:38 +02:00
parent 1b9fa40237
commit 310c5e936d
7 changed files with 61 additions and 39 deletions

View file

@ -20,6 +20,7 @@ redisMaxConnections = 30
[Config] [Config]
hmacKey = "secretkey" # random key for cryptographic signing of video urls hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls
tokenCount = 10 tokenCount = 10
# minimum amount of usable tokens. tokens are used to authorize API requests, # minimum amount of usable tokens. tokens are used to authorize API requests,
# but they expire after ~1 hour, and have a limit of 187 requests. # but they expire after ~1 hour, and have a limit of 187 requests.

View file

@ -20,6 +20,10 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
hostname: cfg.get("Server", "hostname", "nitter.net"), hostname: cfg.get("Server", "hostname", "nitter.net"),
staticDir: cfg.get("Server", "staticDir", "./public"), staticDir: cfg.get("Server", "staticDir", "./public"),
hmacKey: cfg.get("Config", "hmacKey", "secretkey"),
base64Media: cfg.get("Config", "base64Media", false),
minTokens: cfg.get("Config", "tokenCount", 10),
cacheDir: cfg.get("Cache", "directory", "/tmp/nitter"), cacheDir: cfg.get("Cache", "directory", "/tmp/nitter"),
listCacheTime: cfg.get("Cache", "listMinutes", 120), listCacheTime: cfg.get("Cache", "listMinutes", 120),
rssCacheTime: cfg.get("Cache", "rssMinutes", 10), rssCacheTime: cfg.get("Cache", "rssMinutes", 10),
@ -27,10 +31,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
redisHost: cfg.get("Cache", "redisHost", "localhost"), redisHost: cfg.get("Cache", "redisHost", "localhost"),
redisPort: cfg.get("Cache", "redisPort", 6379), redisPort: cfg.get("Cache", "redisPort", 6379),
redisConns: cfg.get("Cache", "redisConnections", 20), redisConns: cfg.get("Cache", "redisConnections", 20),
redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30), redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30)
hmacKey: cfg.get("Config", "hmacKey", "secretkey"),
minTokens: cfg.get("Config", "tokenCount", 10),
) )
return (conf, cfg) return (conf, cfg)

View file

@ -15,6 +15,7 @@ const
nbsp = $Rune(0x000A0) nbsp = $Rune(0x000A0)
wwwRegex = re"https?://(www[0-9]?\.)?" wwwRegex = re"https?://(www[0-9]?\.)?"
m3u8Regex = re"""url="(.+.m3u8)""""
manifestRegex = re"(.+(.ts|.m3u8|.vmap))" manifestRegex = re"(.+(.ts|.m3u8|.vmap))"
userpicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$" userpicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$"
extRegex = re"(\.[A-z]+)$" extRegex = re"(\.[A-z]+)$"
@ -52,6 +53,11 @@ proc replaceUrl*(url: string; prefs: Prefs; absolute=""): string =
if absolute.len > 0: if absolute.len > 0:
result = result.replace("href=\"/", "href=\"https://" & absolute & "/") result = result.replace("href=\"/", "href=\"https://" & absolute & "/")
proc getM3u8Url*(content: string): string =
var m: RegexMatch
if content.find(m3u8Regex, m):
result = content[m.group(0)[0]]
proc proxifyVideo*(manifest: string; proxy: bool): string = proc proxifyVideo*(manifest: string; proxy: bool): string =
proc cb(m: RegexMatch; s: string): string = proc cb(m: RegexMatch; s: string): string =
result = "https://video.twimg.com" & s[m.group(0)[0]] result = "https://video.twimg.com" & s[m.group(0)[0]]

View file

@ -23,6 +23,7 @@ stdout.flushFile
updateDefaultPrefs(fullCfg) updateDefaultPrefs(fullCfg)
setCacheTimes(cfg) setCacheTimes(cfg)
setHmacKey(cfg.hmacKey) setHmacKey(cfg.hmacKey)
setProxyEncoding(cfg.base64Media)
waitFor initRedisPool(cfg) waitFor initRedisPool(cfg)
stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n" stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"

View file

@ -1,17 +1,16 @@
import uri, strutils, httpclient, os, hashes import uri, strutils, httpclient, os, hashes, base64, re
import asynchttpserver, asyncstreams, asyncfile, asyncnet import asynchttpserver, asyncstreams, asyncfile, asyncnet
import jester, regex import jester
import router_utils import router_utils
import ".."/[types, formatters, agents, utils] import ".."/[types, formatters, agents, utils]
import ../views/general import ../views/general
export asynchttpserver, asyncstreams, asyncfile, asyncnet export asynchttpserver, asyncstreams, asyncfile, asyncnet
export httpclient, os, strutils, asyncstreams, regex export httpclient, os, strutils, asyncstreams, base64, re
const const
m3u8Regex* = re"""url="(.+.m3u8)""""
m3u8Mime* = "application/vnd.apple.mpegurl" m3u8Mime* = "application/vnd.apple.mpegurl"
maxAge* = "max-age=604800" maxAge* = "max-age=604800"
@ -60,13 +59,27 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
finally: finally:
client.safeClose() client.safeClose()
template check*(code): untyped =
if code != Http200:
resp code
else:
enableRawMode()
break route
proc decoded*(req: jester.Request; index: int): string =
let
based = req.matches[0].len > 1
encoded = req.matches[index]
if based: decode(encoded)
else: decodeUrl(encoded)
proc createMediaRouter*(cfg: Config) = proc createMediaRouter*(cfg: Config) =
router media: router media:
get "/pic/?": get "/pic/?":
resp Http404 resp Http404
get "/pic/@url": get re"^\/pic\/(enc)?\/?(.+)":
var url = decodeUrl(@"url") var url = decoded(request, 1)
if "twimg.com" notin url: if "twimg.com" notin url:
url.insert(twimg) url.insert(twimg)
if not url.startsWith(https): if not url.startsWith(https):
@ -75,42 +88,32 @@ proc createMediaRouter*(cfg: Config) =
let uri = parseUri(url) let uri = parseUri(url)
cond isTwitterUrl(uri) == true cond isTwitterUrl(uri) == true
enableRawMode() let code = await proxyMedia(request, url)
let code = await proxyMedia(request, $uri) check code
if code == Http200:
enableRawMode()
break route
else:
resp code
get "/video/@sig/@url": get re"^\/video\/(enc)?\/?(.+)\/(.+)$":
cond "http" in @"url" let url = decoded(request, 2)
var url = decodeUrl(@"url") cond "http" in url
let prefs = cookiePrefs()
if getHmac(url) != @"sig": if getHmac(url) != request.matches[1]:
resp showError("Failed to verify signature", cfg) resp showError("Failed to verify signature", cfg)
if ".mp4" in url or ".ts" in url: if ".mp4" in url or ".ts" in url:
let code = await proxyMedia(request, url) let code = await proxyMedia(request, url)
if code == Http200: check code
enableRawMode()
break route
else:
resp code
var content: string var content: string
if ".vmap" in url: if ".vmap" in url:
var m: RegexMatch let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent))
content = await safeFetch(url, mediaAgent) if m3u8.len > 0:
if content.find(m3u8Regex, m):
url = decodeUrl(content[m.group(0)[0]])
content = await safeFetch(url, mediaAgent) content = await safeFetch(url, mediaAgent)
else: else:
resp Http404 resp Http404
if ".m3u8" in url: if ".m3u8" in url:
let vid = await safeFetch(url, mediaAgent) let
vid = await safeFetch(url, mediaAgent)
prefs = cookiePrefs()
content = proxifyVideo(vid, prefs.proxyVideos) content = proxifyVideo(vid, prefs.proxyVideos)
resp content, m3u8Mime resp content, m3u8Mime

View file

@ -201,6 +201,7 @@ type
staticDir*: string staticDir*: string
hmacKey*: string hmacKey*: string
base64Media*: bool
minTokens*: int minTokens*: int
cacheDir*: string cacheDir*: string

View file

@ -1,7 +1,9 @@
import strutils, strformat, sequtils, uri, tables import strutils, strformat, sequtils, uri, tables, base64
import nimcrypto, regex import nimcrypto, regex
var hmacKey = "secretkey" var
hmacKey = "secretkey"
base64Media = false
const const
https* = "https://" https* = "https://"
@ -20,18 +22,25 @@ const
proc setHmacKey*(key: string) = proc setHmacKey*(key: string) =
hmacKey = key hmacKey = key
proc setProxyEncoding*(state: bool) =
base64Media = state
proc getHmac*(data: string): string = proc getHmac*(data: string): string =
($hmac(sha256, hmacKey, data))[0 .. 12] ($hmac(sha256, hmacKey, data))[0 .. 12]
proc getVidUrl*(link: string): string = proc getVidUrl*(link: string): string =
if link.len == 0: return if link.len == 0: return
let let sig = getHmac(link)
sig = getHmac(link) if base64Media:
url = encodeUrl(link) &"/video/enc/{sig}/{encode(link, safe=true)}"
&"/video/{sig}/{url}" else:
&"/video/{sig}/{encodeUrl(link)}"
proc getPicUrl*(link: string): string = proc getPicUrl*(link: string): string =
&"/pic/{encodeUrl(link)}" if base64Media:
&"/pic/enc/{encode(link, safe=true)}"
else:
&"/pic/{encodeUrl(link)}"
proc cleanFilename*(filename: string): string = proc cleanFilename*(filename: string): string =
const reg = re"[^A-Za-z0-9._-]" const reg = re"[^A-Za-z0-9._-]"