From ea1075cb5bc759b9c2039352daa3882954687ee0 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 15 Mar 2020 23:48:37 +0000 Subject: [PATCH] Rewrite file provider to proxy data download (rather than redirect) Reworked the fileprovider to proxy data directly from the provider rather than attempt to funkily redirect when needed, since it was overly complex and wouldn't work well in the long run. Temporarily added "file" as constant return for determining the filetype through Backblaze. Added a Makefile to make life easier as well, and rewrote the README to reflect the refactor/rewrite and new approach to providers. --- Makefile | 10 ++++++++ README.md | 43 ++++++++++++++++++++++++-------- build-pi.sh | 1 - files/backblaze.go | 58 ++++++++++++++++++++++++++++++++++++++++--- files/disk.go | 22 ++++++++++++++-- files/fileprovider.go | 33 ++++++++++++++++-------- router/filerouter.go | 10 ++++++++ 7 files changed, 150 insertions(+), 27 deletions(-) create mode 100644 Makefile delete mode 100755 build-pi.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6659ddf --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +.DEFAULT_GOAL := build + +build: + go build + +pi: + env GOOS=linux GOARCH=arm GOARM=5 go build + +run: + go run webserver.go \ No newline at end of file diff --git a/README.md b/README.md index f7ca090..3f40357 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,38 @@ small go nas platform for my raspberry pi ## usage -``` -cp assets/config/config.example.json assets/config/config.json - # edit config file with your hot/cold storage locations -nano assets/config/config.json -# run -go run webserver.go -# or build and run -go build; ./nas +### configuration + +unlike the initial version of this project, the current build uses _providers_ to determine how to handle various +functions related to files. currently, two are implemented, `disk` and `backblaze`, since they are the primary providers +i use myself. the providers you would like to use can be added to `providers.yml` alongside the binary. + +for example, here is a sample configuration implementing both of them: + +```yaml +disk: + provider: disk + path: /tmp/nas +backblaze: + provider: backblaze + config: + appKeyId: APP_KEY_ID + appId: APP_ID + bucket: BUCKET_ID ``` -you can also optionally use the `build-pi.sh` to build it for a raspberry pi (tested with raspberry pi 3 model b+) +(read more here: [#providers](#providers)) -then navigate to `localhost:3000` +### running + +after adding the providers you would like to use, the application can be run simply with `./nas`. + +### building + +this project uses go modules and a makefile, so building should be relatively straightforward. + + - `make` will build the project for your system's architecture. + - `make pi` will build the project with the `GOOS=linux GOARCH=arm GOARM=5 go` flags set for raspberry pis. ## api @@ -23,6 +42,10 @@ initially the heavy lifting was done by the server, but the need for a better fr full documentation coming soon once actual functionality has been nailed down. +## providers + +// todo + ## credits svg icons via https://iconsvg.xyz diff --git a/build-pi.sh b/build-pi.sh deleted file mode 100755 index 1b38f3e..0000000 --- a/build-pi.sh +++ /dev/null @@ -1 +0,0 @@ -env GOOS=linux GOARCH=arm GOARM=5 go build diff --git a/files/backblaze.go b/files/backblaze.go index 544613f..e66d7d2 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -12,12 +12,14 @@ import ( type BackblazeProvider struct { FileProvider Bucket string + DownloadLocation string } type BackblazeAuthPayload struct { AccountId string `json:"accountId"` AuthToken string `json:"authorizationToken"` ApiUrl string `json:"apiUrl"` + DownloadUrl string `json:"downloadUrl"` } type BackblazeFile struct { @@ -32,6 +34,15 @@ type BackblazeFilePayload struct { Files []BackblazeFile `json:"files"` } +type BackblazeBucketInfo struct { + BucketId string `json:"bucketId"` + BucketName string `json:"bucketName"` +} + +type BackblazeBucketInfoPayload struct { + Buckets []BackblazeBucketInfo `json:"buckets"` +} + // Call Backblaze API endpoint to authorize and gather facts. func (bp *BackblazeProvider) Authorize(appKeyId string, appKey string) error { client := &http.Client{} @@ -60,7 +71,8 @@ func (bp *BackblazeProvider) Authorize(appKeyId string, appKey string) error { } bp.Authentication = data.AuthToken bp.Location = data.ApiUrl - bp.Name = "Backblaze|" + data.AccountId + bp.Name = data.AccountId + bp.DownloadLocation = data.DownloadUrl return nil } @@ -111,10 +123,50 @@ func (bp *BackblazeProvider) GetDirectory(path string) Directory { return finalDir } -func (bp *BackblazeProvider) ViewFile(path string) string { - return "" +func (bp *BackblazeProvider) ViewFile(path string) []byte { + client := &http.Client{} + // Get bucket name >:( + bucketIdPayload := fmt.Sprintf(`{"accountId": "%s", "bucketId": "%s"}`, bp.Name, bp.Bucket) + + req, err := http.NewRequest("POST", bp.Location + "/b2api/v2/b2_list_buckets", + bytes.NewBuffer([]byte(bucketIdPayload))) + req.Header.Add("Authorization", bp.Authentication) + + res, err := client.Do(req) + bucketData, err := ioutil.ReadAll(res.Body) + + var data BackblazeBucketInfoPayload + json.Unmarshal(bucketData, &data) + ourBucket := data.Buckets[0].BucketName + // Get file and write over to client. + url := strings.Join([]string{bp.DownloadLocation,"file",ourBucket,path}, "/") + req, err = http.NewRequest("GET", + url, + bytes.NewBuffer([]byte(""))) + if err != nil { + fmt.Println(err.Error()) + return nil + } + req.Header.Add("Authorization", bp.Authentication) + file, err := client.Do(req) + + if err != nil { + fmt.Println(err.Error()) + return nil + } + + fileBytes, err := ioutil.ReadAll(file.Body) + if err != nil { + fmt.Println(err.Error()) + return nil + } + return fileBytes } func (bp *BackblazeProvider) SaveFile(contents []byte, path string) bool { return true +} + +func (bp *BackblazeProvider) DetermineType(path string) string { + return "file" } \ No newline at end of file diff --git a/files/disk.go b/files/disk.go index f536455..096a08a 100644 --- a/files/disk.go +++ b/files/disk.go @@ -36,8 +36,13 @@ func (dp *DiskProvider) GetDirectory(path string) Directory { } } -func (dp *DiskProvider) ViewFile(path string) string { - return strings.Join([]string{dp.Location,path}, "/") +func (dp *DiskProvider) ViewFile(path string) []byte { + file := strings.Join([]string{dp.Location,path}, "/") + fileContents, err := ioutil.ReadFile(file) + if err != nil { + return nil + } + return fileContents } func (dp *DiskProvider) SaveFile(contents []byte, path string) bool { @@ -46,4 +51,17 @@ func (dp *DiskProvider) SaveFile(contents []byte, path string) bool { return false } return true +} + +func (dp *DiskProvider) DetermineType(path string) string { + rp := strings.Join([]string{dp.Location,path}, "/") + file, err := os.Stat(rp) + if err != nil { + return "" + } + if file.IsDir() { + return "directory" + } + + return "file" } \ No newline at end of file diff --git a/files/fileprovider.go b/files/fileprovider.go index b5be55c..68b9aa9 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -3,15 +3,16 @@ package files import "fmt" type FileProvider struct { - Name string `yaml:"name"` - Authentication string `yaml:"authentication"` - Location string `yaml:"path"` - Config map[string]string `yaml:"config"` + Name string `yaml:"name"` + Provider string `yaml:"provider"` + Authentication string `yaml:"authentication"` + Location string `yaml:"path"` + Config map[string]string `yaml:"config"` } type Directory struct { - Path string - Files []FileInfo + Path string + Files []FileInfo } type FileInfo struct { @@ -22,10 +23,16 @@ type FileInfo struct { var Providers map[string]FileProvider +type FileContents struct { + Content []byte + IsUrl bool +} + type FileProviderInterface interface { GetDirectory(path string) Directory - ViewFile(path string) string + ViewFile(path string) []byte SaveFile(contents []byte, path string) bool + DetermineType(path string) string } func TranslateProvider(codename string, i *FileProviderInterface) { @@ -35,7 +42,7 @@ func TranslateProvider(codename string, i *FileProviderInterface) { return } if codename == "backblaze" { - bbProv := &BackblazeProvider{provider, provider.Config["bucket"]} + bbProv := &BackblazeProvider{provider, provider.Config["bucket"], ""} err := bbProv.Authorize(provider.Config["appKeyId"], provider.Config["appId"]) if err != nil { @@ -52,10 +59,14 @@ func (f FileProvider) GetDirectory(path string) Directory { return Directory{} } -func (f FileProvider) ViewFile(path string) string { - return "" +func (f FileProvider) ViewFile(path string) []byte { + return nil } func (f FileProvider) SaveFile(contents []byte, path string) bool { return false -} \ No newline at end of file +} + +func (f FileProvider) DetermineType(path string) string { + return "" +} diff --git a/router/filerouter.go b/router/filerouter.go index 6b34cac..2bf13da 100644 --- a/router/filerouter.go +++ b/router/filerouter.go @@ -19,6 +19,16 @@ func HandleProvider() common.Handler { fileList := provider.GetDirectory("") if vars["file"] != "" { + fileType := provider.DetermineType(vars["file"]) + if fileType == "" { + w.Write([]byte("file not found")) + return nil + } + if fileType == "file" { + file := provider.ViewFile(vars["file"]) + w.Write(file) + return nil + } fileList = provider.GetDirectory(vars["file"]) } data, err := json.Marshal(fileList)