Break out file host URL to allow custom CDNs

cmd/soju/main.go

@@ -113,6 +113,7 @@ func loadConfig() (*config.Server, *soju.Config, error) {
 		MOTD:                      motd,
 		Auth:                      auth,
 		FileUploader:              fileUploader,
+		FileCdn:                   raw.FileCdn,
 	}
 	return raw, cfg, nil
 }
@@ -151,7 +152,6 @@ func main() {
 	srv := soju.NewServer(db)
 	srv.SetConfig(serverCfg)
 	srv.Logger = soju.NewLogger(log.Writer(), debug)
-
 	fileUploadHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		cfg := srv.Config()
 		h := fileupload.Handler{
@@ -159,6 +159,7 @@ func main() {
 			DB:          db,
 			Auth:        cfg.Auth,
 			HTTPOrigins: cfg.HTTPOrigins,
+			Cdn:         cfg.FileCdn,
 		}
 		h.ServeHTTP(w, r)
 	})

config/config.go

@@ -83,6 +83,7 @@ type Server struct {
 	MsgStore   MsgStore
 	Auth       Auth
 	FileUpload *FileUpload
+	FileCdn    string
 
 	HTTPOrigins    []string
 	HTTPIngress    string
@@ -130,6 +131,7 @@ func Load(filename string) (*Server, error) {
 		Log                 []string   `scfg:"log"`
 		Auth                []string   `scfg:"auth"`
 		FileUpload          []string   `scfg:"file-upload"`
+		FileCdn             string     `scfg:"file-cdn"`
 		HTTPOrigin          []string   `scfg:"http-origin"`
 		HTTPIngress         string     `scfg:"http-ingress"`
 		AcceptProxyIP       []string   `scfg:"accept-proxy-ip"`
@@ -214,11 +216,17 @@ func Load(filename string) (*Server, error) {
 			if source == "" {
 				return nil, fmt.Errorf("directive file-upload: driver %q requires a source", driver)
 			}
+		case "s3":
+			if source == "" {
+				return nil, fmt.Errorf("directive file-upload: driver %q requires a source", driver)
+			}
+
 		default:
 			return nil, fmt.Errorf("directive file-upload: unknown driver %q", driver)
 		}
 		srv.FileUpload = &FileUpload{driver, source}
 	}
+	srv.FileCdn = raw.FileCdn
 	for _, origin := range raw.HTTPOrigin {
 		if _, err := path.Match(origin, origin); err != nil {
 			return nil, fmt.Errorf("directive http-origin: %v", err)

fileupload/fileupload.go

@@ -58,13 +58,15 @@ var primaryExts = map[string]string{
 
 type Uploader interface {
 	load(filename string) (basename string, modTime time.Time, content io.ReadSeekCloser, err error)
-	store(r io.Reader, username, mimeType, basename string) (outFilename string, err error)
+	store(r io.Reader, username, mimeType, basename string, contentLength int64) (outFilename string, err error)
 }
 
 func New(driver, source string) (Uploader, error) {
 	switch driver {
 	case "fs":
 		return &fs{source}, nil
+	case "s3":
+		return &s3{source}, nil
 	default:
 		return nil, fmt.Errorf("unknown file upload driver %q", driver)
 	}
@@ -75,6 +77,7 @@ type Handler struct {
 	Auth        auth.Authenticator
 	DB          database.Database
 	HTTPOrigins []string
+	Cdn         string
 }
 
 func (h *Handler) checkOrigin(reqOrigin string) bool {
@@ -151,6 +154,12 @@ func (h *Handler) fetch(resp http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	if h.Cdn != "" {
+		resp.Header().Set("Location", h.Cdn + "/" + filename)
+		resp.WriteHeader(http.StatusCreated)
+		return
+	}
+
 	basename, modTime, content, err := h.Uploader.load(filename)
 	if err != nil {
 		http.Error(resp, "failed to open file", http.StatusNotFound)
@@ -276,13 +285,18 @@ func (h *Handler) store(resp http.ResponseWriter, req *http.Request) {
 	}
 
 	r := &limitedReader{r: req.Body, n: maxSize}
-	outFilename, err := h.Uploader.store(r, username, mimeType, basename)
+	outFilename, err := h.Uploader.store(r, username, mimeType, basename, req.ContentLength)
 	if err != nil {
+		fmt.Println(err)
 		http.Error(resp, "failed to write file", http.StatusInternalServerError)
 		return
 	}
 
-	resp.Header().Set("Location", "/uploads/"+outFilename)
+	if h.Cdn != "" {
+		resp.Header().Set("Location", h.Cdn + "/" + outFilename)
+	} else {
+		resp.Header().Set("Location", "/uploads/"+outFilename)
+	}
 	resp.WriteHeader(http.StatusCreated)
 }
 

fileupload/fs.go

@@ -40,7 +40,7 @@ func (fs *fs) load(filename string) (basename string, modTime time.Time, content
 	return basename, fi.ModTime(), f, nil
 }
 
-func (fs *fs) store(r io.Reader, username, mimeType, origBasename string) (outFilename string, err error) {
+func (fs *fs) store(r io.Reader, username, mimeType, origBasename string, contentLength int64) (outFilename string, err error) {
 	origBasename = filepath.Base(origBasename)
 
 	var suffix string

fileupload/s3.go

@@ -0,0 +1,55 @@
+package fileupload
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/google/uuid"
+)
+
+type s3 struct {
+	endpoint string
+}
+
+// load implements Uploader.
+func (s *s3) load(filename string) (basename string, modTime time.Time, content io.ReadSeekCloser, err error) {
+	panic("unimplemented")
+}
+
+// store implements Uploader.
+func (s *s3) store(r io.Reader, username string, mimeType string, basename string, contentLength int64) (outFilename string, err error) {
+	split := strings.Split(s.endpoint, "/")
+	bucket := split[len(split)-1]
+	filename := url.PathEscape(username) + "/" + uuid.NewString()
+
+	endpointResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		return aws.Endpoint{
+			URL: s.endpoint,
+		}, nil
+	})
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithEndpointResolverWithOptions(endpointResolver),
+		config.WithRegion("auto"),
+	)
+
+	s3client := awss3.NewFromConfig(cfg)
+	_, err = s3client.PutObject(context.TODO(), &awss3.PutObjectInput{
+		Bucket: &bucket,
+		Key:    &filename,
+		ContentType: &mimeType,
+		ContentLength: &contentLength,
+		Body:   r,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to upload to s3 api: %v", err)
+	}
+	
+	return filename, nil
+}

go.mod

@@ -21,6 +21,29 @@ require (
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.30.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.23 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+)
+
+require (
+	github.com/aws/aws-sdk-go v1.54.11
+	github.com/aws/aws-sdk-go-v2/config v1.27.23
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.57.1
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

@@ -7,6 +7,44 @@ git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9 h1:Ahny8Ud1LjVMMA
 git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA=
 github.com/SherClockHolmes/webpush-go v1.3.0 h1:CAu3FvEE9QS4drc3iKNgpBWFfGqNthKlZhp5QpYnu6k=
 github.com/SherClockHolmes/webpush-go v1.3.0/go.mod h1:AxRHmJuYwKGG1PVgYzToik1lphQvDnqFYDqimHvwhIw=
+github.com/aws/aws-sdk-go v1.54.11 h1:Zxuv/R+IVS0B66yz4uezhxH9FN9/G2nbxejYqAMFjxk=
+github.com/aws/aws-sdk-go v1.54.11/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o=
+github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
+github.com/aws/aws-sdk-go-v2/config v1.27.23 h1:Cr/gJEa9NAS7CDAjbnB7tHYb3aLZI2gVggfmSAasDac=
+github.com/aws/aws-sdk-go-v2/config v1.27.23/go.mod h1:WMMYHqLCFu5LH05mFOF5tsq1PGEMfKbu083VKqLCd0o=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.23 h1:G1CfmLVoO2TdQ8z9dW+JBc/r8+MqyPQhXCafNZcXVZo=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.23/go.mod h1:V/DvSURn6kKgcuKEk4qwSwb/fZ2d++FFARtWSbXnLqY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 h1:THZJJ6TU/FOiM7DZFnisYV9d49oxXWUzsVIMTuf3VNU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13/go.mod h1:VISUTg6n+uBaYIWPBaIG0jk7mbBxm7DUqBtU2cUDDWI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 h1:2jyRZ9rVIMisyQRnhSS/SqlckveoxXneIumECVFP91Y=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15/go.mod h1:bDRG3m382v1KJBk1cKz7wIajg87/61EiiymEyfLvAe0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 h1:Eq2THzHt6P41mpjS2sUzz/3dJYFRqdWZ+vQaEMm98EM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13/go.mod h1:FgwTca6puegxgCInYwGjmd4tB9195Dd6LCuA+8MjpWw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.57.1 h1:aHPtNY87GZ214N4rShgIo+5JQz7ICrJ50i17JbueUTw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.57.1/go.mod h1:hdV0NTYd0RwV4FvNKhKUNbPLZoq9CTr/lke+3I7aCAI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1 h1:lCEv9f8f+zJ8kcFeAjRZsekLd/x5SAm96Cva+VbUdo8=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -26,6 +64,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -109,6 +151,7 @@ google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/irc.v4 v4.0.0 h1:5jsLkU2Tg+R2nGNqmkGCrciasyi4kNkDXhyZD+C31yY=
 gopkg.in/irc.v4 v4.0.0/go.mod h1:BfjDz9MmuWW6OZY7iq4naOhudO8+QQCdO4Ko18jcsRE=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

server.go

@@ -152,6 +152,7 @@ type Config struct {
 	EnableUsersOnAuth         bool
 	Auth                      auth.Authenticator
 	FileUploader              fileupload.Uploader
+	FileCdn                   string
 }
 
 type Server struct {