From bf1f06b79c750e1258928cfb301a52d05defad2a Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Mon, 24 Feb 2020 18:07:47 +0000 Subject: [PATCH] Refactoring file storage to enable different providers Expanding this as we go, currently have POC Backblaze B2 support and basic 'disk' provider as well. Still WIP, but functional for the most part. Also moving to simplified YAML configuration. Overall, simplifying things to be extensible down the line. Still work to be done, but coming along nicely. --- .gitignore | 4 +- assets/config/config.example.json | 4 - assets/web/index.html | 1 - files/backblaze.go | 120 ++++++++++++++++++ files/disk.go | 49 ++++++++ files/fileprovider.go | 61 ++++++++++ files/files.go | 194 ------------------------------ go.mod | 11 +- go.sum | 14 ++- router/filerouter.go | 41 +++++++ router/router.go | 37 ++---- system/system.go | 74 ------------ webserver.go | 13 ++ 13 files changed, 315 insertions(+), 308 deletions(-) delete mode 100644 assets/config/config.example.json delete mode 100644 assets/web/index.html create mode 100644 files/backblaze.go create mode 100644 files/disk.go create mode 100644 files/fileprovider.go delete mode 100644 files/files.go create mode 100644 router/filerouter.go delete mode 100644 system/system.go diff --git a/.gitignore b/.gitignore index 505b3e7..c2466e6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,8 @@ # IntelliJ .idea/ -# Config file -config.json +# Config files +providers.yml # Binary nas diff --git a/assets/config/config.example.json b/assets/config/config.example.json deleted file mode 100644 index db55697..0000000 --- a/assets/config/config.example.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "ColdStorage": "", - "HotStorage": "" -} \ No newline at end of file diff --git a/assets/web/index.html b/assets/web/index.html deleted file mode 100644 index 74e5d5d..0000000 --- a/assets/web/index.html +++ /dev/null @@ -1 +0,0 @@ -React App
\ No newline at end of file diff --git a/files/backblaze.go b/files/backblaze.go new file mode 100644 index 0000000..544613f --- /dev/null +++ b/files/backblaze.go @@ -0,0 +1,120 @@ +package files + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" +) + +type BackblazeProvider struct { + FileProvider + Bucket string +} + +type BackblazeAuthPayload struct { + AccountId string `json:"accountId"` + AuthToken string `json:"authorizationToken"` + ApiUrl string `json:"apiUrl"` +} + +type BackblazeFile struct { + Action string `json:"action"` + Size int `json:"contentLength"` + Type string `json:"contentType"` + FileName string `json:"fileName"` + Timestamp int `json:"uploadTimestamp"` +} + +type BackblazeFilePayload struct { + Files []BackblazeFile `json:"files"` +} + +// Call Backblaze API endpoint to authorize and gather facts. +func (bp *BackblazeProvider) Authorize(appKeyId string, appKey string) error { + client := &http.Client{} + req, err := http.NewRequest("GET", + "https://api.backblazeb2.com/b2api/v2/b2_authorize_account", + nil) + if err != nil { + return err + } + req.SetBasicAuth(appKeyId, appKey) + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + var data BackblazeAuthPayload + + err = json.Unmarshal(body, &data) + if err != nil { + return err + } + bp.Authentication = data.AuthToken + bp.Location = data.ApiUrl + bp.Name = "Backblaze|" + data.AccountId + + return nil +} + +func (bp *BackblazeProvider) GetDirectory(path string) Directory { + client := &http.Client{} + + requestBody := fmt.Sprintf(`{"bucketId": "%s"}`, bp.Bucket) + + req, err := http.NewRequest("POST", + bp.Location + "/b2api/v2/b2_list_file_names", + bytes.NewBuffer([]byte(requestBody))) + if err != nil { + return Directory{} + } + req.Header.Add("Authorization", bp.Authentication) + resp, err := client.Do(req) + if err != nil { + return Directory{} + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return Directory{} + } + + var data BackblazeFilePayload + err = json.Unmarshal(body, &data) + if err != nil { + fmt.Println(err.Error()) + return Directory{} + } + finalDir := Directory{ + Path: bp.Bucket, + } + for _, v := range data.Files { + file := FileInfo{ + IsDirectory: v.Action == "folder", + Name: v.FileName, + } + if v.Action != "folder" { + split := strings.Split(v.FileName, ".") + file.Extension = split[len(split) - 1] + } + finalDir.Files = append(finalDir.Files, file) + } + + return finalDir +} + +func (bp *BackblazeProvider) ViewFile(path string) string { + return "" +} + +func (bp *BackblazeProvider) SaveFile(contents []byte, path string) bool { + return true +} \ No newline at end of file diff --git a/files/disk.go b/files/disk.go new file mode 100644 index 0000000..f536455 --- /dev/null +++ b/files/disk.go @@ -0,0 +1,49 @@ +package files + +import ( + "io/ioutil" + "os" + "strings" +) + +type DiskProvider struct{ + FileProvider +} + +func (dp *DiskProvider) GetDirectory(path string) Directory { + rp := strings.Join([]string{dp.Location,path}, "/") + fileDir, err := ioutil.ReadDir(rp) + if err != nil { + _ = os.MkdirAll(path, 0644) + } + var fileList []FileInfo + + for _, file := range fileDir { + info := FileInfo{ + IsDirectory: file.IsDir(), + Name: file.Name(), + } + if !info.IsDirectory { + split := strings.Split(file.Name(), ".") + info.Extension = split[len(split) - 1] + } + fileList = append(fileList, info) + } + + return Directory{ + Path: rp, + Files: fileList, + } +} + +func (dp *DiskProvider) ViewFile(path string) string { + return strings.Join([]string{dp.Location,path}, "/") +} + +func (dp *DiskProvider) SaveFile(contents []byte, path string) bool { + err := ioutil.WriteFile(path, contents, 0600) + if err != nil { + return false + } + return true +} \ No newline at end of file diff --git a/files/fileprovider.go b/files/fileprovider.go new file mode 100644 index 0000000..b5be55c --- /dev/null +++ b/files/fileprovider.go @@ -0,0 +1,61 @@ +package files + +import "fmt" + +type FileProvider struct { + Name string `yaml:"name"` + Authentication string `yaml:"authentication"` + Location string `yaml:"path"` + Config map[string]string `yaml:"config"` +} + +type Directory struct { + Path string + Files []FileInfo +} + +type FileInfo struct { + IsDirectory bool + Name string + Extension string +} + +var Providers map[string]FileProvider + +type FileProviderInterface interface { + GetDirectory(path string) Directory + ViewFile(path string) string + SaveFile(contents []byte, path string) bool +} + +func TranslateProvider(codename string, i *FileProviderInterface) { + provider := Providers[codename] + if codename == "disk" { + *i = &DiskProvider{provider,} + return + } + if codename == "backblaze" { + bbProv := &BackblazeProvider{provider, provider.Config["bucket"]} + + err := bbProv.Authorize(provider.Config["appKeyId"], provider.Config["appId"]) + if err != nil { + fmt.Println(err.Error()) + } + *i = bbProv + return + } + *i = FileProvider{} +} + +/** DO NOT USE THESE DEFAULTS **/ +func (f FileProvider) GetDirectory(path string) Directory { + return Directory{} +} + +func (f FileProvider) ViewFile(path string) string { + return "" +} + +func (f FileProvider) SaveFile(contents []byte, path string) bool { + return false +} \ No newline at end of file diff --git a/files/files.go b/files/files.go deleted file mode 100644 index 8d33b89..0000000 --- a/files/files.go +++ /dev/null @@ -1,194 +0,0 @@ -package files - -import ( - "encoding/json" - "fmt" - "github.com/gmemstr/nas/auth" - "github.com/gmemstr/nas/common" - "github.com/gorilla/mux" - "io" - "io/ioutil" - "net/http" - "os" - "strings" -) - -type Config struct { - ColdStorage string - HotStorage string -} - -type Directory struct { - Path string - Files []FileInfo - Previous string - Prefix string - SinglePrefix string -} - -type FileInfo struct { - IsDirectory bool - Name string -} - -func GetUserDirectory(r *http.Request, tier string) (string, string, string) { - usr, err := auth.DecryptCookie(r) - if err != nil { - return "", "", "" - } - - username := usr.Username - - d, err := ioutil.ReadFile("assets/config/config.json") - if err != nil { - panic(err) - } - - var config Config - err = json.Unmarshal(d, &config) - if err != nil { - panic(err) - } - - // Default to hot storage - storage := config.HotStorage + username - prefix := "files" - singleprefix := "file" - if tier == "cold" { - storage = config.ColdStorage + username - prefix = "archive" - singleprefix = "archived" - } - - // Ensure directory exists. - _, err = ioutil.ReadDir(storage) - if err != nil && storage == "" { - fmt.Println(storage) - _ = os.MkdirAll(storage, 0644) - } - - return storage, prefix, singleprefix -} - -// Lists out directory using template. -func Listing() common.Handler { - - return func(rc *common.RouterContext, w http.ResponseWriter, r *http.Request) *common.HTTPError { - vars := mux.Vars(r) - id := vars["file"] - tier := vars["tier"] - storage, prefix, singleprefix := GetUserDirectory(r, tier) - if storage == "" && prefix == "" && singleprefix == "" { - http.Redirect(w, r, "/", http.StatusTemporaryRedirect) - return &common.HTTPError{ - Message: "Unauthorized, or unable to find cookie", - StatusCode: http.StatusTemporaryRedirect, - } - } - path := storage - if id != "" { - path = storage + id - } - - fileDir, err := ioutil.ReadDir(path) - if err != nil && path == "" { - fmt.Println(path) - _ = os.MkdirAll(path, 0644) - } - var fileList []FileInfo - - for _, file := range fileDir { - info := FileInfo{ - IsDirectory: file.IsDir(), - Name: file.Name(), - } - fileList = append(fileList, info) - } - path = strings.Replace(path, storage, "", -1) - - // Figure out what our previous location was. - previous := strings.Split(path, "/") - previous = previous[:len(previous)-1] - previousPath := strings.Join(previous, "/") - - directory := Directory{ - Path: path, - Files: fileList, - Previous: previousPath, - Prefix: prefix, - SinglePrefix: singleprefix, - } - - resultJson, err := json.Marshal(directory); - w.Write(resultJson); - return nil; - } - -} - -// Lists out directory using template. -func ViewFile() common.Handler { - - return func(rc *common.RouterContext, w http.ResponseWriter, r *http.Request) *common.HTTPError { - vars := mux.Vars(r) - id := vars["file"] - tier := vars["tier"] - - d, err := ioutil.ReadFile("assets/config/config.json") - if err != nil { - panic(err) - } - - var config Config - err = json.Unmarshal(d, &config) - if err != nil { - panic(err) - } - // Default to hot storage - storage, _, _ := GetUserDirectory(r, tier) - path := storage + "/" + id - - common.ReadAndServeFile(path, w) - return nil - } - -} - -func UploadFile() common.Handler { - - return func(rc *common.RouterContext, w http.ResponseWriter, r *http.Request) *common.HTTPError { - d, err := ioutil.ReadFile("assets/config/config.json") - - vars := mux.Vars(r) - tier := vars["tier"] - - var config Config - err = json.Unmarshal(d, &config) - if err != nil { - panic(err) - } - - err = r.ParseMultipartForm(32 << 20) - path := strings.Join(r.Form["path"], "") - - // Default to hot storage - storage, _, _ := GetUserDirectory(r, tier) - - file, handler, err := r.FormFile("file") - if err != nil { - fmt.Println(err) - return nil - } - defer file.Close() - - f, err := os.OpenFile(storage+"/"+path+"/"+handler.Filename, os.O_WRONLY|os.O_CREATE, 0666) - - if err != nil { - panic(err) - } - defer f.Close() - io.Copy(f, file) - return nil - } - -} diff --git a/go.mod b/go.mod index 465e8a2..40f5d1c 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,13 @@ -module gmem.ca/nas +module github.com/gmemstr/nas go 1.13 require ( - github.com/gmemstr/nas v0.0.0-20190728044305-652472635722 - github.com/gorilla/mux v1.7.4 // indirect - github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect + github.com/go-yaml/yaml v2.1.0+incompatible + github.com/gorilla/mux v1.7.4 + github.com/kr/pretty v0.2.0 // indirect + github.com/mattn/go-sqlite3 v2.0.3+incompatible golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index 8331bcd..9932cbc 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,12 @@ -github.com/gmemstr/nas v0.0.0-20190728044305-652472635722 h1:LmCeaQfQHTfKtx1HbR9cHMndomz14InE0d5lBWp/opI= -github.com/gmemstr/nas v0.0.0-20190728044305-652472635722/go.mod h1:hv1O7aXobFKTuw2JKHkzEnFMdQqv6wcLEgoTD7mxIY4= +github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -11,3 +16,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/router/filerouter.go b/router/filerouter.go new file mode 100644 index 0000000..6b34cac --- /dev/null +++ b/router/filerouter.go @@ -0,0 +1,41 @@ +package router + +import ( + "encoding/json" + "github.com/gmemstr/nas/common" + "github.com/gmemstr/nas/files" + "github.com/gorilla/mux" + "net/http" +) + +func HandleProvider() common.Handler { + + return func(rc *common.RouterContext, w http.ResponseWriter, r *http.Request) *common.HTTPError { + vars := mux.Vars(r) + if r.Method == "GET" { + providerCodename := vars["provider"] + var provider files.FileProviderInterface + files.TranslateProvider(providerCodename, &provider) + + fileList := provider.GetDirectory("") + if vars["file"] != "" { + fileList = provider.GetDirectory(vars["file"]) + } + data, err := json.Marshal(fileList) + if err != nil { + w.Write([]byte("An error occurred")) + return nil + } + w.Write(data) + } + + return nil + } +} + +func ListProviders() common.Handler { + + return func(rc *common.RouterContext, w http.ResponseWriter, r *http.Request) *common.HTTPError { + return nil + } +} diff --git a/router/router.go b/router/router.go index b6c9f95..4955a50 100644 --- a/router/router.go +++ b/router/router.go @@ -9,8 +9,6 @@ import ( "net/http" "github.com/gmemstr/nas/common" - "github.com/gmemstr/nas/files" - "github.com/gmemstr/nas/system" "github.com/gorilla/mux" ) @@ -49,34 +47,19 @@ func Init() *mux.Router { loginHandler(), )).Methods("POST", "GET") - r.Handle("/api/diskusage", Handle( + r.Handle("/api/providers", Handle( auth.RequireAuthorization(1), - system.DiskUsages(), + ListProviders(), + )).Methods("GET") + + r.Handle(`/api/files/{provider}`, Handle( + //auth.RequireAuthorization(1), + HandleProvider(), )).Methods("GET") - r.Handle(`/api/{tier:(?:hot|cold)}/file/{file:[a-zA-Z0-9=\-\/\s.,&_+]+}`, Handle( - auth.RequireAuthorization(1), - files.ViewFile(), - )).Methods("GET") - - r.Handle("/api/upload/hot", Handle( - auth.RequireAuthorization(1), - files.UploadFile(), - )).Methods("POST") - - r.Handle("/api/upload/{tier:(?:hot|cold)}", Handle( - auth.RequireAuthorization(1), - files.UploadFile(), - )).Methods("POST") - - r.Handle("/api/{tier:(?:hot|cold)}/", Handle( - auth.RequireAuthorization(1), - files.Listing(), - )).Methods("GET") - - r.Handle(`/api/{tier:(?:hot|cold)}/{file:[a-zA-Z0-9=\-\/\s.,&_+]+}`, Handle( - auth.RequireAuthorization(1), - files.Listing(), + r.Handle(`/api/files/{provider}/{file:[a-zA-Z0-9=\-\/\s.,&_+]+}`, Handle( + //auth.RequireAuthorization(1), + HandleProvider(), )).Methods("GET") return r diff --git a/system/system.go b/system/system.go deleted file mode 100644 index 2a3d101..0000000 --- a/system/system.go +++ /dev/null @@ -1,74 +0,0 @@ -package system - -import ( - "encoding/json" - "github.com/gmemstr/nas/common" - "github.com/gmemstr/nas/files" - "io/ioutil" - "net/http" - "os" - "syscall" -) - -type Config struct { - ColdStorage string - HotStorage string -} - -type UsageStat struct { - Available int64 - Free int64 - Total int64 -} - -type UsageStats struct { - ColdStorage UsageStat - HotStorage UsageStat -} - -func DiskUsages() common.Handler { - - return func(rc *common.RouterContext, w http.ResponseWriter, r *http.Request) *common.HTTPError { - var statHot syscall.Statfs_t - var statCold syscall.Statfs_t - - d, err := ioutil.ReadFile("assets/config/config.json") - if err != nil { - panic(err) - } - - var config Config - err = json.Unmarshal(d, &config) - if err != nil { - panic(err) - } - - storage, _, _ := files.GetUserDirectory(r,"hot") - err = syscall.Statfs(storage, &statHot) - if err != nil { - _ = os.MkdirAll(storage, 0644) - } - hotStats := UsageStat{ - Free: statHot.Bsize * int64(statHot.Bfree), - Total: statHot.Bsize * int64(statHot.Blocks), - } - - storage, _, _ = files.GetUserDirectory(r,"cold") - err = syscall.Statfs(storage, &statCold) - if err != nil { - _ = os.MkdirAll(storage, 0644) - } - coldStats := UsageStat{ - Free: statCold.Bsize * int64(statCold.Bfree), - Total: statCold.Bsize * int64(statCold.Blocks), - } - usages := UsageStats{ - HotStorage: hotStats, - ColdStorage: coldStats, - } - // Available blocks * size per block = available space in bytes - resultJson, err := json.Marshal(usages) - w.Write(resultJson) - return nil - } -} \ No newline at end of file diff --git a/webserver.go b/webserver.go index b4afcdb..332e19f 100644 --- a/webserver.go +++ b/webserver.go @@ -11,7 +11,10 @@ import ( "database/sql" "encoding/base64" "fmt" + "github.com/gmemstr/nas/files" + "github.com/go-yaml/yaml" "golang.org/x/crypto/bcrypt" + "io/ioutil" "log" "net/http" "os" @@ -26,6 +29,16 @@ func main() { createLockFile() } + file, err := ioutil.ReadFile("providers.yml") + if err != nil { + panic(err) + } + err = yaml.Unmarshal(file, &files.Providers) + if err != nil { + panic(err) + } + fmt.Println(files.Providers) + r := router.Init() fmt.Println("Your NAS instance is live on port :3000") log.Fatal(http.ListenAndServe(":3000", r))