From 54fe5f9c5c7f37e557b5fae4205a96c6c77dc59a Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 14:43:27 +0100 Subject: [PATCH 01/18] Simplify file serving from local or remote. Prior, we would decide how to serve a file based on whether it was local or remote at a router level. This moves everything to an io.Copy call in the router with the Provider returning an io.Reader. --- files/backblaze.go | 30 +++++++++++++++--------------- files/disk.go | 22 +++++++++++++++++----- files/fileprovider.go | 4 ++-- router/filerouter.go | 42 ++++++++++++++++++++---------------------- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/files/backblaze.go b/files/backblaze.go index 881ffb2..5495911 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "io/ioutil" + "mime" "net/http" + "path/filepath" "strings" ) @@ -142,7 +144,7 @@ func (bp *BackblazeProvider) FilePath(path string) string { return "" } -func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { +func (bp *BackblazeProvider) SendFile(path string, w io.Writer) (stream io.Reader, contenttype string, err error) { client := &http.Client{} // Get bucket name >:( bucketIdPayload := fmt.Sprintf(`{"accountId": "%s", "bucketId": "%s"}`, bp.Name, bp.Bucket) @@ -150,20 +152,17 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { req, err := http.NewRequest("POST", bp.Location + "/b2api/v2/b2_list_buckets", bytes.NewBuffer([]byte(bucketIdPayload))) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } req.Header.Add("Authorization", bp.Authentication) res, err := client.Do(req) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } bucketData, err := ioutil.ReadAll(res.Body) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } var data BackblazeBucketInfoPayload @@ -175,22 +174,23 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { url, bytes.NewBuffer([]byte(""))) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } req.Header.Add("Authorization", bp.Authentication) file, err := client.Do(req) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } - _, err = io.Copy(w, file.Body) - if err != nil { - fmt.Println(err.Error()) - return + contenttype = mime.TypeByExtension(filepath.Ext(path)) + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(file.Body, buf[:]) + contenttype = http.DetectContentType(buf[:n]) } + + return file.Body, contenttype, err } func (bp *BackblazeProvider) SaveFile(file io.Reader, filename string, path string) bool { diff --git a/files/disk.go b/files/disk.go index 3b3e048..1c04c67 100644 --- a/files/disk.go +++ b/files/disk.go @@ -4,7 +4,10 @@ import ( "fmt" "io" "io/ioutil" + "mime" + "net/http" "os" + "path/filepath" "strings" ) @@ -42,13 +45,22 @@ func (dp *DiskProvider) GetDirectory(path string) Directory { } } -func (dp *DiskProvider) FilePath(path string) string { +func (dp *DiskProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { rp := strings.Join([]string{dp.Location,path}, "/") - return rp -} + f, err := os.Open(rp) + if err != nil { + return stream, contenttype, err + } -func (dp *DiskProvider) RemoteFile(path string, writer io.Writer) { - return + contenttype = mime.TypeByExtension(filepath.Ext(rp)) + + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(f, buf[:]) + contenttype = http.DetectContentType(buf[:n]) + } + + return f, contenttype, nil } func (dp *DiskProvider) SaveFile(file io.Reader, filename string, path string) bool { diff --git a/files/fileprovider.go b/files/fileprovider.go index 94aea3d..ebf3a61 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -39,7 +39,7 @@ type FileProviderInterface interface { Setup(args map[string]string) (ok bool) GetDirectory(path string) (directory Directory) FilePath(path string) (realpath string) - RemoteFile(path string, writer io.Writer) + SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) SaveFile(file io.Reader, filename string, path string) (ok bool) ObjectInfo(path string) (exists bool, isDir bool, location string) CreateDirectory(path string) (ok bool) @@ -64,7 +64,7 @@ func (f FileProvider) FilePath(path string) string { } // RemoteFile will bypass http.ServeContent() and instead write directly to the response. -func (f FileProvider) RemoteFile(path string, writer io.Writer) { +func (f FileProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { return } diff --git a/router/filerouter.go b/router/filerouter.go index f127f47..6de343e 100644 --- a/router/filerouter.go +++ b/router/filerouter.go @@ -3,14 +3,14 @@ package router import ( "encoding/json" "fmt" - "github.com/gmemstr/nas/files" - "github.com/gorilla/mux" + "io" "net/http" "net/url" - "os" "sort" "strings" - "time" + + "github.com/gmemstr/nas/files" + "github.com/gorilla/mux" ) func handleProvider() handler { @@ -29,7 +29,7 @@ func handleProvider() handler { StatusCode: http.StatusInternalServerError, } } - ok, isDir, location := provider.ObjectInfo(filename) + ok, isDir, _ := provider.ObjectInfo(filename) if !ok { return &httpError{ Message: fmt.Sprintf("error locating file %s\n", filename), @@ -49,27 +49,25 @@ func handleProvider() handler { } w.Write(data) return nil - } + } else { + stream, contenttype, err := provider.SendFile(filename, w) - // If the file is local, attempt to use http.ServeContent for correct headers. - if location == files.FileIsLocal { - rp := provider.FilePath(filename) - if rp != "" { - f, err := os.Open(rp) - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error opening file %s\n", rp), - StatusCode: http.StatusInternalServerError, - } + if err != nil { + return &httpError{ + Message: fmt.Sprintf("error finding file %s\n", vars["file"]), + StatusCode: http.StatusNotFound, + } + } + w.Header().Set("Content-Type", contenttype) + _, err = io.Copy(w, stream) + if err != nil { + return &httpError{ + Message: fmt.Sprintf("unable to write %s\n", vars["file"]), + StatusCode: http.StatusBadGateway, } - http.ServeContent(w, r, filename, time.Time{}, f) } } - // If the file is remote, then delegate the writing to the response to the provider. - // This isn't a great workaround, but avoids caching the whole file in mem or on disk. - if location == files.FileIsRemote { - provider.RemoteFile(filename, w) - } + return nil } From b90abd87791057cb89b2d5623541a62a1bed6662 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 14:46:28 +0100 Subject: [PATCH 02/18] Remove old file path function. --- files/backblaze.go | 4 ---- files/fileprovider.go | 6 ------ files/files_test.go | 8 -------- 3 files changed, 18 deletions(-) diff --git a/files/backblaze.go b/files/backblaze.go index 5495911..95b2f52 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -140,10 +140,6 @@ func (bp *BackblazeProvider) GetDirectory(path string) Directory { return finalDir } -func (bp *BackblazeProvider) FilePath(path string) string { - return "" -} - func (bp *BackblazeProvider) SendFile(path string, w io.Writer) (stream io.Reader, contenttype string, err error) { client := &http.Client{} // Get bucket name >:( diff --git a/files/fileprovider.go b/files/fileprovider.go index ebf3a61..f7df85c 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -38,7 +38,6 @@ type FileInfo struct { type FileProviderInterface interface { Setup(args map[string]string) (ok bool) GetDirectory(path string) (directory Directory) - FilePath(path string) (realpath string) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) SaveFile(file io.Reader, filename string, path string) (ok bool) ObjectInfo(path string) (exists bool, isDir bool, location string) @@ -58,11 +57,6 @@ func (f FileProvider) GetDirectory(path string) Directory { return Directory{} } -// FilePath returns the path to the file, whether it be a URL or local file path. -func (f FileProvider) FilePath(path string) string { - return "" -} - // RemoteFile will bypass http.ServeContent() and instead write directly to the response. func (f FileProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { return diff --git a/files/files_test.go b/files/files_test.go index 5ccacfd..2fdf8be 100644 --- a/files/files_test.go +++ b/files/files_test.go @@ -24,10 +24,6 @@ func TestFileProvider(t *testing.T) { t.Errorf("Default FileProvider GetDirectory() files returned %v, expected none.", getdirectory.Files) } - filepath := fp.FilePath(""); if filepath != "" { - t.Errorf("Default FileProvider FilePath() %v, expected nothing.", filepath) - } - savefile := fp.SaveFile(nil, "", ""); if savefile != false { t.Errorf("Default FileProvider SaveFile() attempted to save a file.") } @@ -87,10 +83,6 @@ func TestDiskProvider(t *testing.T) { t.Errorf("DiskProvider GetDirectory() files returned %v, expected 1.", getdirectory.Files) } - filepath := dp.FilePath("testing.txt"); if filepath != DISK_TESTING_GROUNDS + "/testing.txt"{ - t.Errorf("DiskProvider FilePath() returned %v, expected path.", filepath) - } - testfile := bytes.NewReader([]byte("second test file!")) savefile := dp.SaveFile(testfile, "second_test.txt", ""); if savefile != true { t.Errorf("DiskProvider SaveFile() could not save a file.") From 53ffe69bdc01f4e9eb91d58cbecaed830b61a81d Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 15:40:20 +0100 Subject: [PATCH 03/18] Trigger CI/CD. From dad608edbcaf35bd1240745c8278a0aa3751e8ab Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 17:31:57 +0100 Subject: [PATCH 04/18] Embed web assets into binary. Theorectically, this means that the entire binary is now self contained, minus the need for the config file. Ripped out the redundant static file directives and root handler while I was at it. --- go.mod | 2 +- router/router.go | 55 +++++------------------------------------------- webserver.go | 8 ++++++- 3 files changed, 13 insertions(+), 52 deletions(-) diff --git a/go.mod b/go.mod index a40e545..7de359e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gmemstr/nas -go 1.13 +go 1.16 require ( github.com/Nerzal/gocloak/v5 v5.1.0 diff --git a/router/router.go b/router/router.go index 3ad6d88..bbf4958 100644 --- a/router/router.go +++ b/router/router.go @@ -1,12 +1,9 @@ package router import ( - "fmt" - "io" "log" "net/http" - "os" - "strconv" + "io/fs" "github.com/gorilla/mux" ) @@ -37,21 +34,10 @@ func handle(handlers ...handler) http.Handler { } // Init initializes the main router and all routes for the application. -func Init() *mux.Router { +func Init(sc fs.FS) *mux.Router { r := mux.NewRouter() - // "Static" paths - r.PathPrefix("/javascript/").Handler(http.StripPrefix("/javascript/", http.FileServer(http.Dir("assets/web/javascript")))) - r.PathPrefix("/css/").Handler(http.StripPrefix("/css/", http.FileServer(http.Dir("assets/web/css")))) - r.PathPrefix("/icons/").Handler(http.StripPrefix("/icons/", http.FileServer(http.Dir("assets/web/icons")))) - - // Paths that require specific handlers - r.Handle("/", handle( - requiresAuth(), - rootHandler(), - )).Methods("GET") - // File & Provider API r.Handle("/api/providers", handle( requiresAuth(), @@ -61,7 +47,7 @@ func Init() *mux.Router { r.Handle(`/api/files/{provider:[a-zA-Z0-9]+\/*}`, handle( requiresAuth(), handleProvider(), - )).Methods("GET", "POST") + )).Methods("GET", "POST", "DELETE") r.Handle(`/api/files/{provider:[a-zA-Z0-9]+}/{file:.+}`, handle( requiresAuth(), @@ -73,38 +59,7 @@ func Init() *mux.Router { callbackAuth(), )).Methods("GET", "POST") + r.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.FS(sc)))) + return r } - -// Handles serving index page. -func rootHandler() handler { - return func(context *requestContext, w http.ResponseWriter, r *http.Request) *httpError { - f, err := os.Open("assets/web/index.html") - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error serving index page from assets/web"), - StatusCode: http.StatusInternalServerError, - } - } - - defer f.Close() - stats, err := f.Stat() - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error serving index page from assets/web"), - StatusCode: http.StatusInternalServerError, - } - } else { - w.Header().Add("Content-Length", strconv.FormatInt(stats.Size(), 10)) - } - - _, err = io.Copy(w, f) - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error serving index page from assets/web"), - StatusCode: http.StatusInternalServerError, - } - } - return nil - } -} diff --git a/webserver.go b/webserver.go index 978bee6..613bf35 100644 --- a/webserver.go +++ b/webserver.go @@ -1,7 +1,9 @@ package main import ( + "embed" "fmt" + "io/fs" "io/ioutil" "log" "net/http" @@ -12,6 +14,9 @@ import ( "github.com/go-yaml/yaml" ) +//go:embed assets/web/* +var sc embed.FS + // Main function that defines routes func main() { // Initialize file providers. @@ -39,7 +44,8 @@ func main() { fmt.Println("Keycloak configured") } - r := router.Init() + fsys, err := fs.Sub(sc, "assets/web") + r := router.Init(fsys) fmt.Println("Your sliproad instance is live on port :3000") log.Fatal(http.ListenAndServe(":3000", r)) } From 81cb0259e5be23cfd901396e2ac1b49090a4d0e0 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 17:33:45 +0100 Subject: [PATCH 05/18] Bump golang image version. --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a835281..8621a8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,12 +4,12 @@ orbs: jobs: build: docker: - - image: cimg/go:1.14 + - image: cimg/go:1.16 steps: - checkout - restore_cache: keys: - - go-mod-{{ checksum "go.sum" }}-v2 + - go-mod-{{ checksum "go.sum" }}-v3 - go-mod-{{ checksum "go.sum" }} - go-mod - upx/install @@ -18,17 +18,17 @@ jobs: - store_artifacts: path: build - save_cache: - key: go-mod-{{ checksum "go.sum" }}-v2 + key: go-mod-{{ checksum "go.sum" }}-v3 paths: - /home/circleci/go/pkg/mod test: docker: - - image: cimg/go:1.14 + - image: cimg/go:1.16 steps: - checkout - restore_cache: keys: - - go-mod-{{ checksum "go.sum" }}-v2 + - go-mod-{{ checksum "go.sum" }}-v3 - go-mod-{{ checksum "go.sum" }} - go-mod - run: From 06dd0f6dec14fb1d2681a87fb624dcdc0cb3b02c Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 23 May 2021 21:04:24 +0100 Subject: [PATCH 06/18] Implement new colour palette. Pink #F06DF2 Black #02060D Dark Blue #081226 Blue #04ADBF Yellow #F2E307 Also fixed minor bug where first 512 bytes of files are sometimes cut off when determining content type. --- Makefile | 5 +- assets/web/css/styles.css | 100 ++++++++++++++++++++--------------- assets/web/javascript/app.js | 20 +++++-- files/disk.go | 4 ++ 4 files changed, 80 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index b303d52..4e24548 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,8 @@ test: go test ./... -cover dist: clean make_build_dir small small_pi - cp -r assets/* build/assets - tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-arm.tar.gz build/assets build/bin/sliproad-arm README.md LICENSE - tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-x86.tar.gz build/assets build/bin/sliproad README.md LICENSE + tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-arm.tar.gz build/bin/sliproad-arm README.md LICENSE + tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-x86.tar.gz build/bin/sliproad README.md LICENSE clean: rm -rf build diff --git a/assets/web/css/styles.css b/assets/web/css/styles.css index 611b274..d38a889 100644 --- a/assets/web/css/styles.css +++ b/assets/web/css/styles.css @@ -1,18 +1,32 @@ -:root { - /* https://coolors.co/fe4a49-fed766-009fb7-e6e6ea-f4f4f8 */ - --orange: rgba(254, 74, 73, 1); - --yellow: rgba(254, 215, 102, 1); - --blue: rgba(0, 159, 183, 1); - --platinum: rgba(230, 230, 234, 1); - --white: rgba(244, 244, 248, 1); +/* +Pink #F06DF2 +Black #02060D +Dark Blue #081226 +Blue #04ADBF +Yellow #F2E307 +*/ +:root { + --black: #02060D; + --blue: #04ADBF; + --dark-blue: #081226; + --yellow: #F2E307; + --pink: #F06DF2; --desktop-width: 1170px; } body, h1 a { font-family: sans-serif; - color: var(--orange); - background-color: var(--white); + color: var(--black); + background-color: var(--black); +} + +.directory > span { + color: white; +} + +h1 a { + color: var(--pink); } .grid-lg { @@ -54,7 +68,7 @@ body, h1 a { position: relative; border-radius: 5px 0 0 5px; display: inline-block; - width: 93%; + width: 100%; } .list a img { @@ -64,42 +78,34 @@ body, h1 a { } .list a.file { - background-color: var(--orange); + background-color: transparent; + border: 1px solid var(--blue); + color: white; } .list a.directory { - background-color: var(--blue); + background-color: var(--dark-blue); } .grid-lg a:visited, .grid-lg a, .list a:visited, .list a { - color: var(--white); + color: white; } .grid-lg a:hover, .list a.directory:hover { - color: var(--blue); - background-color: var(--platinum); + background-color: var(--blue); transition: background-color 0.5s, color 0.5s; } .list a.file:hover { - color: var(--orange); - background-color: var(--platinum); + background-color: var(--dark-blue); transition: background-color 0.5s, color 0.5s; } -@media (prefers-color-scheme: dark) { - body, h1 a { background-color: black; } - .grid-lg a { - color: black; - } - .grid-lg a:visited, .grid-lg a { - color: black - } - .grid-lg a:hover { - color: lightgray; - background-color: darkgray; - transition: background-color 0.5s; - } +.directories { + display: grid; + gap: 2px 2px; + grid-template-columns: 1fr 1fr 1fr 1fr; + grid-template-rows: auto; } @media only screen and (max-width: 1170px) { @@ -127,12 +133,13 @@ input[type="file"] { } input[type="file"] + label, input[type="submit"], button { - color: var(--orange); + color: white; + background-color: transparent; padding: 10px; font-size: 1.25em; font-weight: 700; display: inline-block; - border: 2px solid var(--orange); + border: 2px solid var(--pink); border-radius: 5px; transition: background-color 0.5s, color 0.5s; } @@ -142,33 +149,33 @@ input[type="text"] { font-size: 1.25em; font-weight: 500; display: inline-block; - border: 2px solid var(--orange); + border: 2px solid var(--pink); border-radius: 5px; transition: background-color 0.5s, color 0.5s; width: 50%; + background-color: transparent; + color: white; } input[type="file"]:focus + label, input[type="file"] + label:hover, input[type="submit"]:hover, button:hover { - background-color: var(--orange); - color: var(--white); + color: white; transition: background-color 0.5s, color 0.5s; } progress { - background: var(--platinum); - border: 1px solid var(--orange); + border: 1px solid var(--pink); } progress::-webkit-progress-bar { - background: var(--orange); + background: var(--pink); } progress::-webkit-progress-value { - background: var(--orange); + background: var(--pink); } progress::-moz-progress-bar { - background: var(--orange); + background: var(--pink); } button { @@ -184,10 +191,19 @@ button { } .directory ~ button { - border-color: var(--blue); + border-color: var(--dark-blue); + background-color: white; } .directory ~ button:hover { - background-color: var(--blue); + background-color: grey; +} + +.file ~ button { + border-color: var(--blue); + background-color: white; +} +.file ~ button:hover { + background-color: grey; } .item, .forms { diff --git a/assets/web/javascript/app.js b/assets/web/javascript/app.js index 8f4d0ff..370d2ed 100644 --- a/assets/web/javascript/app.js +++ b/assets/web/javascript/app.js @@ -19,6 +19,9 @@ function getFileListing(provider, path = "") { if (!files) { files = [] } + onlyfiles = files.filter(file => !file.IsDirectory) + onlydirs = files.filter(file => file.IsDirectory) + html`
@@ -30,10 +33,19 @@ function getFileListing(provider, path = "") {
-
- ${files.map(file => - `
- ${file.IsDirectory ? '' : ''}${file.Name} + + +
+ ${onlyfiles.map(file => + ` ` ).join('')} diff --git a/files/disk.go b/files/disk.go index 1c04c67..2549fca 100644 --- a/files/disk.go +++ b/files/disk.go @@ -58,6 +58,10 @@ func (dp *DiskProvider) SendFile(path string, writer io.Writer) (stream io.Reade var buf [512]byte n, _ := io.ReadFull(f, buf[:]) contenttype = http.DetectContentType(buf[:n]) + _, err := f.Seek(0, io.SeekStart) + if err != nil { + return stream, contenttype, err + } } return f, contenttype, nil From 9cd45faf1e4fcb01164af007f11c1e0b88d3d31e Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Wed, 26 May 2021 21:45:16 +0100 Subject: [PATCH 07/18] Minor tweaks. --- assets/web/css/styles.css | 3 +-- assets/web/javascript/app.js | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/assets/web/css/styles.css b/assets/web/css/styles.css index d38a889..c0449df 100644 --- a/assets/web/css/styles.css +++ b/assets/web/css/styles.css @@ -12,7 +12,6 @@ Yellow #F2E307 --dark-blue: #081226; --yellow: #F2E307; --pink: #F06DF2; - --desktop-width: 1170px; } body, h1 a { @@ -45,7 +44,7 @@ h1 a { font-weight: bold; text-decoration: none; transition: background-color 0.5s; - background-color: var(--blue); + background-color: var(--dark-blue); } .list { diff --git a/assets/web/javascript/app.js b/assets/web/javascript/app.js index 370d2ed..8423d26 100644 --- a/assets/web/javascript/app.js +++ b/assets/web/javascript/app.js @@ -44,8 +44,8 @@ function getFileListing(provider, path = "") {
${onlyfiles.map(file => - `
- ${file.Name} + ` ` ).join('')} From 22e1df4bc7fbcb554b5caeeee480e1ba8eed5567 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Fri, 28 May 2021 22:43:30 +0100 Subject: [PATCH 08/18] Starting on basic S3 support. --- files/fileutils.go | 7 +++++ files/s3.go | 78 ++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 16 +++++++++- 4 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 files/s3.go diff --git a/files/fileutils.go b/files/fileutils.go index f30a1ce..170a271 100644 --- a/files/fileutils.go +++ b/files/fileutils.go @@ -17,6 +17,13 @@ func TranslateProvider(codename string, i *FileProviderInterface) { *i = bbProv return } + + if provider.Provider == "s3" { + s3Prov := &S3Provider{provider, provider.Config["region"], provider.Config["bucket"]} + *i = s3Prov + return + } + *i = FileProvider{} } diff --git a/files/s3.go b/files/s3.go new file mode 100644 index 0000000..83db84d --- /dev/null +++ b/files/s3.go @@ -0,0 +1,78 @@ +package files + +import ( + "fmt" + "io" + // I _really_ don't want to deal with AWS API stuff by hand. + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +var svc s3.S3 + +type S3Provider struct { + FileProvider + Region string + Bucket string +} + + +// Setup runs when the application starts up, and allows for things like authentication. +func (s *S3Provider) Setup(args map[string]string) bool { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(s.Region)}, + ) + if err != nil { + return false + } + svc = *s3.New(sess) + return true +} + +// GetDirectory fetches a directory's contents. +func (s *S3Provider) GetDirectory(path string) Directory { + resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.Bucket)}) + if err != nil { + fmt.Println(err) + return Directory{} + } + + dir := Directory{} + for _, item := range resp.Contents { + file := FileInfo{ + IsDirectory: false, + Name: *item.Key, + } + dir.Files = append(dir.Files, file) + } + + return dir +} + +// RemoteFile will bypass http.ServeContent() and instead write directly to the response. +func (s *S3Provider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { + return +} + +// SaveFile will save a file with the contents of the io.Reader at the path specified. +func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool { + return false +} + +// ObjectInfo will return the info for an object given a path to if the file exists and location. +// Should return whether the path exists, if the path is a directory, and if it lives on disk. +// (see constants defined: `FILE_IS_REMOTE` and `FILE_IS_LOCAL`) +func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { + return true, true, "" +} + +// CreateDirectory will create a directory on services that support it. +func (s *S3Provider) CreateDirectory(path string) bool { + return false +} + +// Delete simply deletes a file. This is expected to be a destructive action by default. +func (s *S3Provider) Delete(path string) bool { + return false +} diff --git a/go.mod b/go.mod index 7de359e..8e34ec3 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.16 require ( github.com/Nerzal/gocloak/v5 v5.1.0 + github.com/aws/aws-sdk-go v1.38.51 github.com/go-yaml/yaml v2.1.0+incompatible github.com/gorilla/mux v1.7.4 github.com/kr/pretty v0.2.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index 9094e9c..0e3f6cf 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Nerzal/gocloak/v5 v5.1.0 h1:1YP4+GoY1DZ1k7WyNNr8xbFyt55B9ORn2ZHu+XqUK0Q= github.com/Nerzal/gocloak/v5 v5.1.0/go.mod h1:8v53okuWiWXOKOS6qil8cOn7+5JSQfX1t1d+Nj8FpYk= +github.com/aws/aws-sdk-go v1.38.51 h1:aKQmbVbwOCuQSd8+fm/MR3bq0QOsu9Q7S+/QEND36oQ= +github.com/aws/aws-sdk-go v1.38.51/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -11,6 +13,10 @@ github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwn github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -24,10 +30,18 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 550e722a533ffe66a07af8491172a6a80ab012db Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 10:48:32 +0100 Subject: [PATCH 09/18] Implement SendFile() and ObjectInfo() for S3. Also removed a now useless parameter for file providers, which used to be for writing directly to the HTTP response. --- files/backblaze.go | 2 +- files/disk.go | 2 +- files/fileprovider.go | 4 ++-- files/s3.go | 48 ++++++++++++++++++++++++++++++++++++++++--- router/filerouter.go | 2 +- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/files/backblaze.go b/files/backblaze.go index 95b2f52..d1d2e21 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -140,7 +140,7 @@ func (bp *BackblazeProvider) GetDirectory(path string) Directory { return finalDir } -func (bp *BackblazeProvider) SendFile(path string, w io.Writer) (stream io.Reader, contenttype string, err error) { +func (bp *BackblazeProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { client := &http.Client{} // Get bucket name >:( bucketIdPayload := fmt.Sprintf(`{"accountId": "%s", "bucketId": "%s"}`, bp.Name, bp.Bucket) diff --git a/files/disk.go b/files/disk.go index 2549fca..a0c4b52 100644 --- a/files/disk.go +++ b/files/disk.go @@ -45,7 +45,7 @@ func (dp *DiskProvider) GetDirectory(path string) Directory { } } -func (dp *DiskProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { +func (dp *DiskProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { rp := strings.Join([]string{dp.Location,path}, "/") f, err := os.Open(rp) if err != nil { diff --git a/files/fileprovider.go b/files/fileprovider.go index f7df85c..2073417 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -38,7 +38,7 @@ type FileInfo struct { type FileProviderInterface interface { Setup(args map[string]string) (ok bool) GetDirectory(path string) (directory Directory) - SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) + SendFile(path string) (stream io.Reader, contenttype string, err error) SaveFile(file io.Reader, filename string, path string) (ok bool) ObjectInfo(path string) (exists bool, isDir bool, location string) CreateDirectory(path string) (ok bool) @@ -58,7 +58,7 @@ func (f FileProvider) GetDirectory(path string) Directory { } // RemoteFile will bypass http.ServeContent() and instead write directly to the response. -func (f FileProvider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { +func (f FileProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { return } diff --git a/files/s3.go b/files/s3.go index 83db84d..77b9e80 100644 --- a/files/s3.go +++ b/files/s3.go @@ -3,6 +3,10 @@ package files import ( "fmt" "io" + "mime" + "path/filepath" + "net/http" + // I _really_ don't want to deal with AWS API stuff by hand. "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -10,6 +14,7 @@ import ( ) var svc s3.S3 +var sess *session.Session type S3Provider struct { FileProvider @@ -40,6 +45,15 @@ func (s *S3Provider) GetDirectory(path string) Directory { dir := Directory{} for _, item := range resp.Contents { + ik := *item.Key + // Why is this here? AWS returns a complete list of files, including + // files within subdirectories (prefixed with the dir name). So we can + // ignore directories altogether -- I would prefer to display them but + // not sure what the best method of distinguishing them in ObjectInfo() + // would be. + if ik[len(ik)-1:] == "/" { + continue + } file := FileInfo{ IsDirectory: false, Name: *item.Key, @@ -51,8 +65,23 @@ func (s *S3Provider) GetDirectory(path string) Directory { } // RemoteFile will bypass http.ServeContent() and instead write directly to the response. -func (s *S3Provider) SendFile(path string, writer io.Writer) (stream io.Reader, contenttype string, err error) { - return +func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string, err error) { + req, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &path, + }) + if err != nil { + return stream, contenttype, err + } + + contenttype = mime.TypeByExtension(filepath.Ext(path)) + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(req.Body, buf[:]) + contenttype = http.DetectContentType(buf[:n]) + } + + return req.Body, contenttype, err } // SaveFile will save a file with the contents of the io.Reader at the path specified. @@ -64,7 +93,20 @@ func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool // Should return whether the path exists, if the path is a directory, and if it lives on disk. // (see constants defined: `FILE_IS_REMOTE` and `FILE_IS_LOCAL`) func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { - return true, true, "" + if path == "" { + return true, true, "" + } + + _, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &path, + }) + if err != nil { + fmt.Println(err) + return false, false, "" + } + + return true, false, "" } // CreateDirectory will create a directory on services that support it. diff --git a/router/filerouter.go b/router/filerouter.go index 6de343e..7b38c37 100644 --- a/router/filerouter.go +++ b/router/filerouter.go @@ -50,7 +50,7 @@ func handleProvider() handler { w.Write(data) return nil } else { - stream, contenttype, err := provider.SendFile(filename, w) + stream, contenttype, err := provider.SendFile(filename) if err != nil { return &httpError{ From 8fff6da76b7a497573711a28d6daee1111cdca81 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 19:37:45 +0100 Subject: [PATCH 10/18] Add support for custom endpoints and keys for S3. --- files/fileprovider.go | 2 +- files/fileutils.go | 21 +++++++++++++++++++-- files/s3.go | 38 ++++++++++++++++++++++---------------- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/files/fileprovider.go b/files/fileprovider.go index 2073417..f25da63 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -57,7 +57,7 @@ func (f FileProvider) GetDirectory(path string) Directory { return Directory{} } -// RemoteFile will bypass http.ServeContent() and instead write directly to the response. +// SendFile returns a filestream, a valid MIME type for the file and an error. func (f FileProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { return } diff --git a/files/fileutils.go b/files/fileutils.go index 170a271..f13b1c2 100644 --- a/files/fileutils.go +++ b/files/fileutils.go @@ -19,7 +19,24 @@ func TranslateProvider(codename string, i *FileProviderInterface) { } if provider.Provider == "s3" { - s3Prov := &S3Provider{provider, provider.Config["region"], provider.Config["bucket"]} + s3Prov := &S3Provider{ + FileProvider: provider, + Region: provider.Config["region"], + Bucket: provider.Config["bucket"], + Endpoint: "", + KeyID: "", + KeySecret: "", + } + if _, ok := provider.Config["endpoint"]; ok { + s3Prov.Endpoint = provider.Config["endpoint"] + } + if _, ok := provider.Config["keyid"]; ok { + s3Prov.KeyID = provider.Config["keyid"] + } + if _, ok := provider.Config["keysecret"]; ok { + s3Prov.KeySecret = provider.Config["keysecret"] + } + *i = s3Prov return } @@ -40,4 +57,4 @@ func SetupProviders() { fmt.Printf("%s initialized successfully\n", name) } } -} \ No newline at end of file +} diff --git a/files/s3.go b/files/s3.go index 77b9e80..1b24a69 100644 --- a/files/s3.go +++ b/files/s3.go @@ -4,11 +4,12 @@ import ( "fmt" "io" "mime" - "path/filepath" "net/http" + "path/filepath" // I _really_ don't want to deal with AWS API stuff by hand. "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) @@ -18,16 +19,26 @@ var sess *session.Session type S3Provider struct { FileProvider - Region string - Bucket string + Region string + Bucket string + Endpoint string + KeyID string + KeySecret string } - // Setup runs when the application starts up, and allows for things like authentication. func (s *S3Provider) Setup(args map[string]string) bool { - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(s.Region)}, - ) + config := &aws.Config{Region: aws.String(s.Region)} + if s.KeyID != "" && s.KeySecret != "" { + config = &aws.Config{ + Region: aws.String(s.Region), + Credentials: credentials.NewStaticCredentials(s.KeyID, s.KeySecret, ""), + } + } + if s.Endpoint != "" { + config.Endpoint = &s.Endpoint + } + sess, err := session.NewSession(config) if err != nil { return false } @@ -40,7 +51,7 @@ func (s *S3Provider) GetDirectory(path string) Directory { resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.Bucket)}) if err != nil { fmt.Println(err) - return Directory{} + return Directory{} } dir := Directory{} @@ -56,7 +67,7 @@ func (s *S3Provider) GetDirectory(path string) Directory { } file := FileInfo{ IsDirectory: false, - Name: *item.Key, + Name: *item.Key, } dir.Files = append(dir.Files, file) } @@ -64,11 +75,10 @@ func (s *S3Provider) GetDirectory(path string) Directory { return dir } -// RemoteFile will bypass http.ServeContent() and instead write directly to the response. func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string, err error) { req, err := svc.GetObject(&s3.GetObjectInput{ Bucket: &s.Bucket, - Key: &path, + Key: &path, }) if err != nil { return stream, contenttype, err @@ -84,14 +94,10 @@ func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string return req.Body, contenttype, err } -// SaveFile will save a file with the contents of the io.Reader at the path specified. func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool { return false } -// ObjectInfo will return the info for an object given a path to if the file exists and location. -// Should return whether the path exists, if the path is a directory, and if it lives on disk. -// (see constants defined: `FILE_IS_REMOTE` and `FILE_IS_LOCAL`) func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { if path == "" { return true, true, "" @@ -99,7 +105,7 @@ func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { _, err := svc.GetObject(&s3.GetObjectInput{ Bucket: &s.Bucket, - Key: &path, + Key: &path, }) if err != nil { fmt.Println(err) From 4aae9887e1528c01afde531fa3b1e8cf6bd065fa Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 19:53:03 +0100 Subject: [PATCH 11/18] Fix referencing for S3 providers. S3 providers would overwrite each other depending on the order they're initialized. --- files/s3.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/files/s3.go b/files/s3.go index 1b24a69..eb9b489 100644 --- a/files/s3.go +++ b/files/s3.go @@ -12,10 +12,11 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" ) var svc s3.S3 -var sess *session.Session +var sess session.Session type S3Provider struct { FileProvider @@ -38,11 +39,12 @@ func (s *S3Provider) Setup(args map[string]string) bool { if s.Endpoint != "" { config.Endpoint = &s.Endpoint } - sess, err := session.NewSession(config) + ss, err := session.NewSession(config) + sess = *ss if err != nil { return false } - svc = *s3.New(sess) + svc = *s3.New(&sess) return true } @@ -95,7 +97,17 @@ func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string } func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool { - return false + uploader := s3manager.NewUploader(&sess) + _, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: &s.Bucket, + Key: &filename, + Body: file, + }) + if err != nil { + return false + } + + return true } func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { From e7f3d5402e5ce4587b4f35595d5f220dc0f33cda Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 19:57:49 +0100 Subject: [PATCH 12/18] Deletion support for S3. --- files/s3.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/files/s3.go b/files/s3.go index eb9b489..0774ab3 100644 --- a/files/s3.go +++ b/files/s3.go @@ -99,9 +99,9 @@ func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool { uploader := s3manager.NewUploader(&sess) _, err := uploader.Upload(&s3manager.UploadInput{ - Bucket: &s.Bucket, - Key: &filename, - Body: file, + Bucket: &s.Bucket, + Key: &filename, + Body: file, }) if err != nil { return false @@ -134,5 +134,18 @@ func (s *S3Provider) CreateDirectory(path string) bool { // Delete simply deletes a file. This is expected to be a destructive action by default. func (s *S3Provider) Delete(path string) bool { - return false + _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(path)}) + if err != nil { + return false + } + + err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(s.Bucket), + Key: aws.String(path), + }) + if err != nil { + return false + } + + return true } From 59cfb6656fab19b77182b8e191ea2cdf13499dae Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 20:12:32 +0100 Subject: [PATCH 13/18] Actually fix overlapping S3 providers. --- files/s3.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/files/s3.go b/files/s3.go index 0774ab3..ddd2516 100644 --- a/files/s3.go +++ b/files/s3.go @@ -15,9 +15,6 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" ) -var svc s3.S3 -var sess session.Session - type S3Provider struct { FileProvider Region string @@ -25,6 +22,9 @@ type S3Provider struct { Endpoint string KeyID string KeySecret string + + svc s3.S3 + sess session.Session } // Setup runs when the application starts up, and allows for things like authentication. @@ -40,17 +40,17 @@ func (s *S3Provider) Setup(args map[string]string) bool { config.Endpoint = &s.Endpoint } ss, err := session.NewSession(config) - sess = *ss + s.sess = *ss if err != nil { return false } - svc = *s3.New(&sess) + s.svc = *s3.New(&s.sess) return true } // GetDirectory fetches a directory's contents. func (s *S3Provider) GetDirectory(path string) Directory { - resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.Bucket)}) + resp, err := s.svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.Bucket)}) if err != nil { fmt.Println(err) return Directory{} @@ -78,7 +78,7 @@ func (s *S3Provider) GetDirectory(path string) Directory { } func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string, err error) { - req, err := svc.GetObject(&s3.GetObjectInput{ + req, err := s.svc.GetObject(&s3.GetObjectInput{ Bucket: &s.Bucket, Key: &path, }) @@ -97,7 +97,7 @@ func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string } func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool { - uploader := s3manager.NewUploader(&sess) + uploader := s3manager.NewUploader(&s.sess) _, err := uploader.Upload(&s3manager.UploadInput{ Bucket: &s.Bucket, Key: &filename, @@ -115,7 +115,7 @@ func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { return true, true, "" } - _, err := svc.GetObject(&s3.GetObjectInput{ + _, err := s.svc.GetObject(&s3.GetObjectInput{ Bucket: &s.Bucket, Key: &path, }) @@ -134,12 +134,12 @@ func (s *S3Provider) CreateDirectory(path string) bool { // Delete simply deletes a file. This is expected to be a destructive action by default. func (s *S3Provider) Delete(path string) bool { - _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(path)}) + _, err := s.svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(path)}) if err != nil { return false } - err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + err = s.svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ Bucket: aws.String(s.Bucket), Key: aws.String(path), }) From 636ab32eadeea71c588d8ef5de265fa7bf306a0a Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 22:32:38 +0100 Subject: [PATCH 14/18] Tidy README, remove UPX requirement. --- .circleci/config.yml | 4 +- Makefile | 2 - README.md | 105 +++++++++++++++------------ assets/config_examples/providers.yml | 13 ++++ 4 files changed, 74 insertions(+), 50 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8621a8f..23d11a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,5 @@ version: 2.1 -orbs: - upx: circleci/upx@1.0.1 + jobs: build: docker: @@ -12,7 +11,6 @@ jobs: - go-mod-{{ checksum "go.sum" }}-v3 - go-mod-{{ checksum "go.sum" }} - go-mod - - upx/install - run: command: make dist - store_artifacts: diff --git a/Makefile b/Makefile index 4e24548..6071f6c 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,9 @@ pi: make_build_dir small: make_build_dir go build -o build/bin/sliproad -ldflags="-s -w" - upx --brute build/bin/sliproad -9 --no-progress small_pi: make_build_dir env GOOS=linux GOARCH=arm GOARM=5 go build -o build/bin/sliproad-arm -ldflags="-s -w" - upx --brute build/bin/sliproad-arm -9 --no-progress run: go run webserver.go diff --git a/README.md b/README.md index b306d82..cf085d7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,24 @@ -# sliproad -merging filesystems together +# Sliproad -## about +Merging filesystems together -this project aims to be an easy-to-user web API that allows the management of cloud storage, whether it be on -the host machine or part of a remote api. this is intended mostly for my own use, but i am documenting it in a way that -i hope allows others to pick it up and improve on it down the line. +## About -if something is unclear, feel free to open an issue :) +This project aims to be an easy-to-use web API and frontend that allows the +management of cloud storage, whether it be on the host machine or part of a +remote API, alongside local filesystems. While this is intended mostly for my +own use, I am documenting it in a way that I hope allows others to use it! -## configuration +## Configuration -unlike the initial version of this project, the current build uses _providers_ to determine how to handle various -functions related to files. currently, two are implemented, `disk` and `backblaze`, since they are the primary providers -i use myself. the providers you would like to use can be added to `providers.yml` alongside the binary. +Sliproad uses "Providers" to support various filesystems "types", whether it be +remote or local. Currently, three exist - `disk` for filesystems local to the +machine, `backblaze` to leverage Backblaze B2 file storage and `s3` for AWS S3 +(and other compatible providers). -for example, here is a sample configuration implementing both of them: +An example of leveraging all three, in various forms, can be found below. As +more are added, this example will be updated, and more examples can be found in +the `assets/config_examples` directory. ```yaml disk: @@ -24,53 +27,65 @@ disk: backblaze: provider: backblaze config: - appKeyId: APP_KEY_ID - appId: APP_ID - bucket: BUCKET_ID + bucket: some-bucket + applicationKeyId: application-key-id + applicationKey: application-key +s3: + provider: s3 + config: + region: eu-west-2 + bucket: some-bucket +# An example of an S3 compatible API, doesn't have to be Backblaze. +backblazes3: + provider: s3 + config: + bucket: some-bucket + region: us-west-000 + endpoint: s3.us-west-000.backblazeb2.com + keyid: key-id + keysecret: key-secret ``` -(read more here: [#providers](#providers)) +## Running -## running +After configuring the providers you would like to utilize, simply run +`./sliproad`. This will spin up the webserver at `127.0.0.1:3000`, listening on +all addresses. -after adding the providers you would like to use, the application can be run simply with `./nas`. it will attach to port -`:3000`. +## Building -## building - -this project uses go modules and a makefile, so building should be relatively straightforward. +This project leverages a Makefile to macro common commands for running, testing +and building this project. - `make` will build the project for your system's architecture. - `make run` will run the project with `go run` - - `make pi` will build the project with the `GOOS=linux GOARCH=arm GOARM=5 go` flags set for raspberry pis. - -## providers + - `make pi` will build the project with the `GOOS=linux GOARCH=arm GOARM=5 go` flags set for Raspberry Pi. + - `make dist` will build and package the binaries for distribution. -"providers" provide a handful of functions to interact nicely with a filesystem, whether it be on local disk or on a -remote server via an api. best-effort is done to keep these performant, up to date and minimal. +### Adding Providers -there are a few built-in providers, and more can be added by opening a pull request. +New file providers can be implemented by building off the +`FileProviderInterface` struct, as the existing providers demonstrate. You can +then instruct the [`TranslateProvider()`](https://github.com/gmemstr/sliproad/blob/master/files/fileutils.go#L8-L21) +that it exists and how to configure it. -|name|service|configuration example| -|----|-------|---------------------| -|disk|local filesystem|assets/config_examples/disk.yml| -|backblaze|backblaze b2|assets/config_examples/backblaze.yml| +## Authentication [!] -you can find a full configuration file under `assets/config_examples/providers.yml` +Authentication is a bit tricky and due to be reworked in the next iteration of +this project. Currently, support for [Keycloak](https://www.keycloak.org/) is +implemented, if a bit naively. You can turn this authentication requirement on +by adding `auth.yml` alongside your `providers.yml` file with the following: -### custom provider +```yaml +provider_url: "https://url-of-keycloak" +realm: "keycloak-realm" +redirect_base_url: "https://location-of-sliproad" +``` -custom file providers can be implemented by adding a new go file to the `files` module. it should -implement the `FileProviderInterface` interface. +Keycloak support is not currently actively supported, and is due to be removed +in the next major release of Sliproad. That said, if you encounter any major +bugs utilizing it before this, _please_ open an issue so I can dig in further. -## authentication - -basic authentication support utilizing [keycloak](https://keycloak.org/) has been implemented, but work -is being done to bring this more inline with the storage provider implementation. see `assets/config_examples/auth.yml` -for an example configuration - having this file alongside the binary will activate authentication on all -`/api/files` endpoints. note that this implementation is a work in progress, and a seperate branch will -contain further improvements. - -## icons +## Credits SVG Icons provided by Paweł Kuna: https://github.com/tabler/tabler-icons (see assets/web/icons/README.md) \ No newline at end of file diff --git a/assets/config_examples/providers.yml b/assets/config_examples/providers.yml index 5c19b86..780b7c3 100644 --- a/assets/config_examples/providers.yml +++ b/assets/config_examples/providers.yml @@ -18,3 +18,16 @@ backblaze: applicationKeyId: aaaaaaaaaaaa applicationKey: aaaaaaaaaaaa bucket: aaaaaaaaaaaa +s3: + provider: s3 + config: + region: eu-west-2 + bucket: some-bucket +backblazes3: + provider: s3 + config: + bucket: sliproad-testing + region: us-west-000 + endpoint: s3.us-west-000.backblazeb2.com + keyid: key-id + keysecret: key-secret \ No newline at end of file From cbd30f91554ed681ac67f1bdd324de615f1955bf Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 22:42:24 +0100 Subject: [PATCH 15/18] Clean up CCI config. --- .circleci/config.yml | 5 +++-- README.md | 6 ++++++ assets/config_examples/README.md | 0 3 files changed, 9 insertions(+), 2 deletions(-) delete mode 100644 assets/config_examples/README.md diff --git a/.circleci/config.yml b/.circleci/config.yml index 23d11a3..5904a84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,8 +32,9 @@ jobs: - run: command: make test workflows: - version: 2 build-and-test: jobs: - - build - test + - build: + requires: + - test diff --git a/README.md b/README.md index cf085d7..9fe3fe0 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ After configuring the providers you would like to utilize, simply run `./sliproad`. This will spin up the webserver at `127.0.0.1:3000`, listening on all addresses. +## API + +This project is largely API-first, and documentation can be found here: + +https://github.com/gmemstr/sliproad/wiki/API + ## Building This project leverages a Makefile to macro common commands for running, testing diff --git a/assets/config_examples/README.md b/assets/config_examples/README.md deleted file mode 100644 index e69de29..0000000 From 60b69213ae3e22fabde94f9d346e53c9c3b25e58 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 22:44:31 +0100 Subject: [PATCH 16/18] Redundant redundancy. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9fe3fe0..affd4b2 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ Merging filesystems together ## About This project aims to be an easy-to-use web API and frontend that allows the -management of cloud storage, whether it be on the host machine or part of a -remote API, alongside local filesystems. While this is intended mostly for my -own use, I am documenting it in a way that I hope allows others to use it! +management of cloud storage alongside local filesystems. While this is intended +mostly for my own use, I am documenting it in a way that I hope allows others +to use it! ## Configuration From c8a898d861b7fcd72a55e51e48eded8936117c72 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 22:49:17 +0100 Subject: [PATCH 17/18] Add frontend blurb. --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index affd4b2..01fefac 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,17 @@ After configuring the providers you would like to utilize, simply run `./sliproad`. This will spin up the webserver at `127.0.0.1:3000`, listening on all addresses. +## Frontend + +The frontend is a very lightweight JavaScript application and aims to be very +functional, if a bit rough around the edges. + +![Screenshot_2021-05-29 Sliproad](https://user-images.githubusercontent.com/1878840/120085420-d63cbc80-c0cf-11eb-9fbb-b0b05a3f5d58.png) + +It should scale reasonably well for smaller devices. Because it's now bundled +into the binary (as opposed to distributed alongside), it's no longer possible +to swap it out for a custom frontend without serving the frontend seperately. + ## API This project is largely API-first, and documentation can be found here: @@ -94,4 +105,4 @@ bugs utilizing it before this, _please_ open an issue so I can dig in further. ## Credits -SVG Icons provided by Paweł Kuna: https://github.com/tabler/tabler-icons (see assets/web/icons/README.md) \ No newline at end of file +SVG Icons provided by Paweł Kuna: https://github.com/tabler/tabler-icons (see assets/web/icons/README.md) From e888ca1759b90009a23225c5fc01222ac987a980 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 29 May 2021 22:51:23 +0100 Subject: [PATCH 18/18] Fix typo. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 01fefac..56ba1b1 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ and building this project. New file providers can be implemented by building off the `FileProviderInterface` struct, as the existing providers demonstrate. You can -then instruct the [`TranslateProvider()`](https://github.com/gmemstr/sliproad/blob/master/files/fileutils.go#L8-L21) +then instruct the [`TranslateProvider()`](https://github.com/gmemstr/sliproad/blob/master/files/fileutils.go#L8-L21) function that it exists and how to configure it. ## Authentication [!]