From 54fe5f9c5c7f37e557b5fae4205a96c6c77dc59a Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 14:43:27 +0100 Subject: [PATCH 1/3] Simplify file serving from local or remote. Prior, we would decide how to serve a file based on whether it was local or remote at a router level. This moves everything to an io.Copy call in the router with the Provider returning an io.Reader. --- files/backblaze.go | 30 +++++++++++++++--------------- files/disk.go | 22 +++++++++++++++++----- files/fileprovider.go | 4 ++-- router/filerouter.go | 42 ++++++++++++++++++++---------------------- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/files/backblaze.go b/files/backblaze.go index 881ffb2..5495911 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "io/ioutil" + "mime" "net/http" + "path/filepath" "strings" ) @@ -142,7 +144,7 @@ func (bp *BackblazeProvider) FilePath(path string) string { return "" } -func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { +func (bp *BackblazeProvider) SendFile(path string, w io.Writer) (stream io.Reader, contenttype string, err error) { client := &http.Client{} // Get bucket name >:( bucketIdPayload := fmt.Sprintf(`{"accountId": "%s", "bucketId": "%s"}`, bp.Name, bp.Bucket) @@ -150,20 +152,17 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { req, err := http.NewRequest("POST", bp.Location + "/b2api/v2/b2_list_buckets", bytes.NewBuffer([]byte(bucketIdPayload))) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } req.Header.Add("Authorization", bp.Authentication) res, err := client.Do(req) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } bucketData, err := ioutil.ReadAll(res.Body) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } var data BackblazeBucketInfoPayload @@ -175,22 +174,23 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { url, bytes.NewBuffer([]byte(""))) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } req.Header.Add("Authorization", bp.Authentication) file, err := client.Do(req) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } - _, err = io.Copy(w, file.Body) - if err != nil { - fmt.Println(err.Error()) - return + contenttype = mime.TypeByExtension(filepath.Ext(path)) + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(file.Body, buf[:]) + contenttype = http.DetectContentType(buf[:n]) } + + return file.Body, contenttype, err } func (bp *BackblazeProvider) SaveFile(file io.Reader, filename string, path string) bool { diff --git a/files/disk.go b/files/disk.go index 3b3e048..1c04c67 100644 --- a/files/disk.go +++ b/files/disk.go @@ -4,7 +4,10 @@ import ( "fmt" "io" "io/ioutil" + "mime" + "net/http" "os" + "path/filepath" "strings" ) @@ -42,13 +45,22 @@ func (dp *DiskProvider) GetDirectory(path string) Directory { } } -func (dp *DiskProvider) FilePath(path string) string { +func (dp *DiskProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { rp := strings.Join([]string{dp.Location,path}, "/") - return rp -} + f, err := os.Open(rp) + if err != nil { + return stream, contenttype, err + } -func (dp *DiskProvider) RemoteFile(path string, writer io.Writer) { - return + contenttype = mime.TypeByExtension(filepath.Ext(rp)) + + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(f, buf[:]) + contenttype = http.DetectContentType(buf[:n]) + } + + return f, contenttype, nil } func (dp *DiskProvider) SaveFile(file io.Reader, filename string, path string) bool { diff --git a/files/fileprovider.go b/files/fileprovider.go index 94aea3d..ebf3a61 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -39,7 +39,7 @@ type FileProviderInterface interface { Setup(args map[string]string) (ok bool) GetDirectory(path string) (directory Directory) FilePath(path string) (realpath string) - RemoteFile(path string, writer io.Writer) + SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) SaveFile(file io.Reader, filename string, path string) (ok bool) ObjectInfo(path string) (exists bool, isDir bool, location string) CreateDirectory(path string) (ok bool) @@ -64,7 +64,7 @@ func (f FileProvider) FilePath(path string) string { } // RemoteFile will bypass http.ServeContent() and instead write directly to the response. -func (f FileProvider) RemoteFile(path string, writer io.Writer) { +func (f FileProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { return } diff --git a/router/filerouter.go b/router/filerouter.go index f127f47..6de343e 100644 --- a/router/filerouter.go +++ b/router/filerouter.go @@ -3,14 +3,14 @@ package router import ( "encoding/json" "fmt" - "github.com/gmemstr/nas/files" - "github.com/gorilla/mux" + "io" "net/http" "net/url" - "os" "sort" "strings" - "time" + + "github.com/gmemstr/nas/files" + "github.com/gorilla/mux" ) func handleProvider() handler { @@ -29,7 +29,7 @@ func handleProvider() handler { StatusCode: http.StatusInternalServerError, } } - ok, isDir, location := provider.ObjectInfo(filename) + ok, isDir, _ := provider.ObjectInfo(filename) if !ok { return &httpError{ Message: fmt.Sprintf("error locating file %s\n", filename), @@ -49,27 +49,25 @@ func handleProvider() handler { } w.Write(data) return nil - } + } else { + stream, contenttype, err := provider.SendFile(filename, w) - // If the file is local, attempt to use http.ServeContent for correct headers. - if location == files.FileIsLocal { - rp := provider.FilePath(filename) - if rp != "" { - f, err := os.Open(rp) - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error opening file %s\n", rp), - StatusCode: http.StatusInternalServerError, - } + if err != nil { + return &httpError{ + Message: fmt.Sprintf("error finding file %s\n", vars["file"]), + StatusCode: http.StatusNotFound, + } + } + w.Header().Set("Content-Type", contenttype) + _, err = io.Copy(w, stream) + if err != nil { + return &httpError{ + Message: fmt.Sprintf("unable to write %s\n", vars["file"]), + StatusCode: http.StatusBadGateway, } - http.ServeContent(w, r, filename, time.Time{}, f) } } - // If the file is remote, then delegate the writing to the response to the provider. - // This isn't a great workaround, but avoids caching the whole file in mem or on disk. - if location == files.FileIsRemote { - provider.RemoteFile(filename, w) - } + return nil } From b90abd87791057cb89b2d5623541a62a1bed6662 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 14:46:28 +0100 Subject: [PATCH 2/3] Remove old file path function. --- files/backblaze.go | 4 ---- files/fileprovider.go | 6 ------ files/files_test.go | 8 -------- 3 files changed, 18 deletions(-) diff --git a/files/backblaze.go b/files/backblaze.go index 5495911..95b2f52 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -140,10 +140,6 @@ func (bp *BackblazeProvider) GetDirectory(path string) Directory { return finalDir } -func (bp *BackblazeProvider) FilePath(path string) string { - return "" -} - func (bp *BackblazeProvider) SendFile(path string, w io.Writer) (stream io.Reader, contenttype string, err error) { client := &http.Client{} // Get bucket name >:( diff --git a/files/fileprovider.go b/files/fileprovider.go index ebf3a61..f7df85c 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -38,7 +38,6 @@ type FileInfo struct { type FileProviderInterface interface { Setup(args map[string]string) (ok bool) GetDirectory(path string) (directory Directory) - FilePath(path string) (realpath string) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) SaveFile(file io.Reader, filename string, path string) (ok bool) ObjectInfo(path string) (exists bool, isDir bool, location string) @@ -58,11 +57,6 @@ func (f FileProvider) GetDirectory(path string) Directory { return Directory{} } -// FilePath returns the path to the file, whether it be a URL or local file path. -func (f FileProvider) FilePath(path string) string { - return "" -} - // RemoteFile will bypass http.ServeContent() and instead write directly to the response. func (f FileProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { return diff --git a/files/files_test.go b/files/files_test.go index 5ccacfd..2fdf8be 100644 --- a/files/files_test.go +++ b/files/files_test.go @@ -24,10 +24,6 @@ func TestFileProvider(t *testing.T) { t.Errorf("Default FileProvider GetDirectory() files returned %v, expected none.", getdirectory.Files) } - filepath := fp.FilePath(""); if filepath != "" { - t.Errorf("Default FileProvider FilePath() %v, expected nothing.", filepath) - } - savefile := fp.SaveFile(nil, "", ""); if savefile != false { t.Errorf("Default FileProvider SaveFile() attempted to save a file.") } @@ -87,10 +83,6 @@ func TestDiskProvider(t *testing.T) { t.Errorf("DiskProvider GetDirectory() files returned %v, expected 1.", getdirectory.Files) } - filepath := dp.FilePath("testing.txt"); if filepath != DISK_TESTING_GROUNDS + "/testing.txt"{ - t.Errorf("DiskProvider FilePath() returned %v, expected path.", filepath) - } - testfile := bytes.NewReader([]byte("second test file!")) savefile := dp.SaveFile(testfile, "second_test.txt", ""); if savefile != true { t.Errorf("DiskProvider SaveFile() could not save a file.") From 53ffe69bdc01f4e9eb91d58cbecaed830b61a81d Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 15:40:20 +0100 Subject: [PATCH 3/3] Trigger CI/CD.