Proxy media instead of using file cache

This commit is contained in:
Zed 2020-06-06 04:39:22 +02:00
parent 1dcb191903
commit 68a5ac20b6
7 changed files with 80 additions and 61 deletions

View file

@ -1,4 +1,4 @@
import asyncdispatch, mimetypes import asyncdispatch
from net import Port from net import Port
import jester import jester

View file

@ -1,4 +1,5 @@
import asyncfile, uri, strutils, httpclient, os, mimetypes import uri, strutils, httpclient, os, hashes
import asynchttpserver, asyncstreams, asyncfile, asyncnet
import jester, regex import jester, regex
@ -6,12 +7,59 @@ import router_utils
import ".."/[types, formatters, agents] import ".."/[types, formatters, agents]
import ../views/general import ../views/general
export asyncfile, httpclient, os, strutils export asynchttpserver, asyncstreams, asyncfile, asyncnet
export regex export httpclient, os, strutils, asyncstreams, regex
const
m3u8Regex* = re"""url="(.+.m3u8)""""
m3u8Mime* = "application/vnd.apple.mpegurl"
maxAge* = "max-age=604800"
const m3u8Regex* = re"""url="(.+.m3u8)""""
let mediaAgent* = getAgent() let mediaAgent* = getAgent()
template respond*(req: asynchttpserver.Request; headers) =
var msg = "HTTP/1.1 200 OK\c\L"
for k, v in headers:
msg.add(k & ": " & v & "\c\L")
msg.add "\c\L"
yield req.client.send(msg)
proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
result = Http200
let
request = req.getNativeReq()
client = newAsyncHttpClient(userAgent=mediaAgent)
try:
let res = await client.get(url)
if res.status != "200 OK":
return Http404
let hashed = $hash(url)
if request.headers.getOrDefault("If-None-Match") == hashed:
return Http304
let headers = newHttpHeaders({
"Content-Type": res.headers["content-type", 0],
"Content-Length": res.headers["content-length", 0],
"Cache-Control": maxAge,
"ETag": hashed
})
respond(request, headers)
var (hasValue, data) = (true, "")
while hasValue:
(hasValue, data) = await res.bodyStream.read()
if hasValue:
await request.client.send(data)
data.setLen 0
except HttpRequestError, OSError:
result = Http404
finally:
client.safeClose()
proc createMediaRouter*(cfg: Config) = proc createMediaRouter*(cfg: Config) =
router media: router media:
get "/pic/?": get "/pic/?":
@ -24,48 +72,13 @@ proc createMediaRouter*(cfg: Config) =
let uri = parseUri(decodeUrl(@"url")) let uri = parseUri(decodeUrl(@"url"))
cond isTwitterUrl($uri) == true cond isTwitterUrl($uri) == true
let path = uri.path.split("/")[2 .. ^1].join("/") enableRawMode()
let filename = cfg.cacheDir / cleanFilename(path & uri.query) let code = await proxyMedia(request, $uri)
if code == Http200:
if path.len == 0: enableRawMode()
resp Http404 break route
else:
if not existsDir(cfg.cacheDir): resp code
createDir(cfg.cacheDir)
if not existsFile(filename):
let client = newAsyncHttpClient(userAgent=mediaAgent)
var failed = false
try:
await client.downloadFile($uri, filename)
except HttpRequestError:
failed = true
removeFile(filename)
except OSError as e:
echo "Disk full, or network error: ", e.msg
failed = true
finally:
client.safeClose()
if failed:
resp Http404
sendFile(filename)
get "/gif/@url":
cond "http" in @"url"
cond "twimg" in @"url"
cond "mp4" in @"url" or "gif" in @"url"
let url = decodeUrl(@"url")
cond isTwitterUrl(url) == true
let content = await safeFetch(url, mediaAgent)
if content.len == 0: resp Http404
let filename = parseUri(url).path.split(".")[^1]
resp content, settings.mimes.getMimetype(filename)
get "/video/@sig/@url": get "/video/@sig/@url":
cond "http" in @"url" cond "http" in @"url"
@ -75,17 +88,26 @@ proc createMediaRouter*(cfg: Config) =
if getHmac(url) != @"sig": if getHmac(url) != @"sig":
resp showError("Failed to verify signature", cfg) resp showError("Failed to verify signature", cfg)
var content = await safeFetch(url, mediaAgent) if ".mp4" in url or ".ts" in url:
if content.len == 0: resp Http404 let code = await proxyMedia(request, url)
if code == Http200:
enableRawMode()
break route
else:
resp code
var content: string
if ".vmap" in url: if ".vmap" in url:
var m: RegexMatch var m: RegexMatch
discard content.find(m3u8Regex, m)
url = decodeUrl(content[m.group(0)[0]])
content = await safeFetch(url, mediaAgent) content = await safeFetch(url, mediaAgent)
if content.find(m3u8Regex, m):
url = decodeUrl(content[m.group(0)[0]])
content = await safeFetch(url, mediaAgent)
else:
resp Http404
if ".m3u8" in url: if ".m3u8" in url:
content = proxifyVideo(content, prefs.proxyVideos) let vid = await safeFetch(url, mediaAgent)
content = proxifyVideo(vid, prefs.proxyVideos)
let ext = parseUri(url).path.split(".")[^1] resp content, m3u8Mime
resp content, settings.mimes.getMimetype(ext)

View file

@ -45,7 +45,7 @@ proc createStatusRouter*(cfg: Config) =
video = getVideoEmbed(cfg, conv.tweet.id) video = getVideoEmbed(cfg, conv.tweet.id)
elif conv.tweet.gif.isSome(): elif conv.tweet.gif.isSome():
images = @[get(conv.tweet.gif).thumb] images = @[get(conv.tweet.gif).thumb]
video = getGifUrl(get(conv.tweet.gif).url) video = getPicUrl(get(conv.tweet.gif).url)
let html = renderConversation(conv, prefs, getPath() & "#m") let html = renderConversation(conv, prefs, getPath() & "#m")
resp renderMain(html, request, cfg, title, desc, resp renderMain(html, request, cfg, title, desc,

View file

@ -35,7 +35,7 @@ proc expired(token: Token): bool {.inline.} =
result = token.init < getTime() - expirationTime result = token.init < getTime() - expirationTime
proc isLimited(token: Token): bool {.inline.} = proc isLimited(token: Token): bool {.inline.} =
token == nil or token.remaining <= 1 and token.reset > getTime() or token == nil or (token.remaining <= 1 and token.reset > getTime()) or
token.expired token.expired
proc release*(token: Token) = proc release*(token: Token) =

View file

@ -28,9 +28,6 @@ proc getVidUrl*(link: string): string =
url = encodeUrl(link) url = encodeUrl(link)
&"/video/{sig}/{url}" &"/video/{sig}/{url}"
proc getGifUrl*(link: string): string =
&"/gif/{encodeUrl(link)}"
proc getPicUrl*(link: string): string = proc getPicUrl*(link: string): string =
&"/pic/{encodeUrl(link)}" &"/pic/{encodeUrl(link)}"

View file

@ -36,7 +36,7 @@
<img src="https://${hostname}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" /> <img src="https://${hostname}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
#elif tweet.gif.isSome: #elif tweet.gif.isSome:
#let thumb = &"https://{hostname}{getPicUrl(get(tweet.gif).thumb)}" #let thumb = &"https://{hostname}{getPicUrl(get(tweet.gif).thumb)}"
#let url = &"https://{hostname}{getGifUrl(get(tweet.gif).url)}" #let url = &"https://{hostname}{getPicUrl(get(tweet.gif).url)}"
<video poster="${thumb}" autoplay muted loop style="max-width:250px;"> <video poster="${thumb}" autoplay muted loop style="max-width:250px;">
<source src="${url}" type="video/mp4"</source></video> <source src="${url}" type="video/mp4"</source></video>
#end if #end if

View file

@ -107,7 +107,7 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
tdiv(class="gallery-gif", style={maxHeight: "unset"}): tdiv(class="gallery-gif", style={maxHeight: "unset"}):
tdiv(class="attachment"): tdiv(class="attachment"):
let thumb = getPicUrl(gif.thumb) let thumb = getPicUrl(gif.thumb)
let url = getGifUrl(gif.url) let url = getPicUrl(gif.url)
if prefs.autoplayGifs: if prefs.autoplayGifs:
video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""): video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
source(src=url, `type`="video/mp4") source(src=url, `type`="video/mp4")