diff --git a/files/backblaze.go b/files/backblaze.go index 95b2f52..d1d2e21 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -140,7 +140,7 @@ func (bp *BackblazeProvider) GetDirectory(path string) Directory { return finalDir } -func (bp *BackblazeProvider) SendFile(path string, w io.Writer) (stream io.Reader, contenttype string, err error) { +func (bp *BackblazeProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { client := &http.Client{} // Get bucket name >:( bucketIdPayload := fmt.Sprintf(`{"accountId": "%s", "bucketId": "%s"}`, bp.Name, bp.Bucket) diff --git a/files/disk.go b/files/disk.go index 2549fca..a0c4b52 100644 --- a/files/disk.go +++ b/files/disk.go @@ -45,7 +45,7 @@ func (dp *DiskProvider) GetDirectory(path string) Directory { } } -func (dp *DiskProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { +func (dp *DiskProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { rp := strings.Join([]string{dp.Location,path}, "/") f, err := os.Open(rp) if err != nil { diff --git a/files/fileprovider.go b/files/fileprovider.go index f7df85c..f25da63 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -38,7 +38,7 @@ type FileInfo struct { type FileProviderInterface interface { Setup(args map[string]string) (ok bool) GetDirectory(path string) (directory Directory) - SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) + SendFile(path string) (stream io.Reader, contenttype string, err error) SaveFile(file io.Reader, filename string, path string) (ok bool) ObjectInfo(path string) (exists bool, isDir bool, location string) CreateDirectory(path string) (ok bool) @@ -57,8 +57,8 @@ func (f FileProvider) GetDirectory(path string) Directory { return Directory{} } -// RemoteFile will bypass http.ServeContent() and instead write directly to the response. -func (f FileProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { +// SendFile returns a filestream, a valid MIME type for the file and an error. +func (f FileProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { return } diff --git a/files/fileutils.go b/files/fileutils.go index f30a1ce..f13b1c2 100644 --- a/files/fileutils.go +++ b/files/fileutils.go @@ -17,6 +17,30 @@ func TranslateProvider(codename string, i *FileProviderInterface) { *i = bbProv return } + + if provider.Provider == "s3" { + s3Prov := &S3Provider{ + FileProvider: provider, + Region: provider.Config["region"], + Bucket: provider.Config["bucket"], + Endpoint: "", + KeyID: "", + KeySecret: "", + } + if _, ok := provider.Config["endpoint"]; ok { + s3Prov.Endpoint = provider.Config["endpoint"] + } + if _, ok := provider.Config["keyid"]; ok { + s3Prov.KeyID = provider.Config["keyid"] + } + if _, ok := provider.Config["keysecret"]; ok { + s3Prov.KeySecret = provider.Config["keysecret"] + } + + *i = s3Prov + return + } + *i = FileProvider{} } @@ -33,4 +57,4 @@ func SetupProviders() { fmt.Printf("%s initialized successfully\n", name) } } -} \ No newline at end of file +} diff --git a/files/s3.go b/files/s3.go new file mode 100644 index 0000000..ddd2516 --- /dev/null +++ b/files/s3.go @@ -0,0 +1,151 @@ +package files + +import ( + "fmt" + "io" + "mime" + "net/http" + "path/filepath" + + // I _really_ don't want to deal with AWS API stuff by hand. + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type S3Provider struct { + FileProvider + Region string + Bucket string + Endpoint string + KeyID string + KeySecret string + + svc s3.S3 + sess session.Session +} + +// Setup runs when the application starts up, and allows for things like authentication. +func (s *S3Provider) Setup(args map[string]string) bool { + config := &aws.Config{Region: aws.String(s.Region)} + if s.KeyID != "" && s.KeySecret != "" { + config = &aws.Config{ + Region: aws.String(s.Region), + Credentials: credentials.NewStaticCredentials(s.KeyID, s.KeySecret, ""), + } + } + if s.Endpoint != "" { + config.Endpoint = &s.Endpoint + } + ss, err := session.NewSession(config) + s.sess = *ss + if err != nil { + return false + } + s.svc = *s3.New(&s.sess) + return true +} + +// GetDirectory fetches a directory's contents. +func (s *S3Provider) GetDirectory(path string) Directory { + resp, err := s.svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.Bucket)}) + if err != nil { + fmt.Println(err) + return Directory{} + } + + dir := Directory{} + for _, item := range resp.Contents { + ik := *item.Key + // Why is this here? AWS returns a complete list of files, including + // files within subdirectories (prefixed with the dir name). So we can + // ignore directories altogether -- I would prefer to display them but + // not sure what the best method of distinguishing them in ObjectInfo() + // would be. + if ik[len(ik)-1:] == "/" { + continue + } + file := FileInfo{ + IsDirectory: false, + Name: *item.Key, + } + dir.Files = append(dir.Files, file) + } + + return dir +} + +func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string, err error) { + req, err := s.svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &path, + }) + if err != nil { + return stream, contenttype, err + } + + contenttype = mime.TypeByExtension(filepath.Ext(path)) + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(req.Body, buf[:]) + contenttype = http.DetectContentType(buf[:n]) + } + + return req.Body, contenttype, err +} + +func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool { + uploader := s3manager.NewUploader(&s.sess) + _, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: &s.Bucket, + Key: &filename, + Body: file, + }) + if err != nil { + return false + } + + return true +} + +func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { + if path == "" { + return true, true, "" + } + + _, err := s.svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &path, + }) + if err != nil { + fmt.Println(err) + return false, false, "" + } + + return true, false, "" +} + +// CreateDirectory will create a directory on services that support it. +func (s *S3Provider) CreateDirectory(path string) bool { + return false +} + +// Delete simply deletes a file. This is expected to be a destructive action by default. +func (s *S3Provider) Delete(path string) bool { + _, err := s.svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(path)}) + if err != nil { + return false + } + + err = s.svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(s.Bucket), + Key: aws.String(path), + }) + if err != nil { + return false + } + + return true +} diff --git a/go.mod b/go.mod index 7de359e..8e34ec3 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.16 require ( github.com/Nerzal/gocloak/v5 v5.1.0 + github.com/aws/aws-sdk-go v1.38.51 github.com/go-yaml/yaml v2.1.0+incompatible github.com/gorilla/mux v1.7.4 github.com/kr/pretty v0.2.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index 9094e9c..0e3f6cf 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Nerzal/gocloak/v5 v5.1.0 h1:1YP4+GoY1DZ1k7WyNNr8xbFyt55B9ORn2ZHu+XqUK0Q= github.com/Nerzal/gocloak/v5 v5.1.0/go.mod h1:8v53okuWiWXOKOS6qil8cOn7+5JSQfX1t1d+Nj8FpYk= +github.com/aws/aws-sdk-go v1.38.51 h1:aKQmbVbwOCuQSd8+fm/MR3bq0QOsu9Q7S+/QEND36oQ= +github.com/aws/aws-sdk-go v1.38.51/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -11,6 +13,10 @@ github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwn github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -24,10 +30,18 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/router/filerouter.go b/router/filerouter.go index 6de343e..7b38c37 100644 --- a/router/filerouter.go +++ b/router/filerouter.go @@ -50,7 +50,7 @@ func handleProvider() handler { w.Write(data) return nil } else { - stream, contenttype, err := provider.SendFile(filename, w) + stream, contenttype, err := provider.SendFile(filename) if err != nil { return &httpError{