diff --git a/.circleci/config.yml b/.circleci/config.yml index a835281..5904a84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,41 +1,40 @@ version: 2.1 -orbs: - upx: circleci/upx@1.0.1 + jobs: build: docker: - - image: cimg/go:1.14 + - image: cimg/go:1.16 steps: - checkout - restore_cache: keys: - - go-mod-{{ checksum "go.sum" }}-v2 + - go-mod-{{ checksum "go.sum" }}-v3 - go-mod-{{ checksum "go.sum" }} - go-mod - - upx/install - run: command: make dist - store_artifacts: path: build - save_cache: - key: go-mod-{{ checksum "go.sum" }}-v2 + key: go-mod-{{ checksum "go.sum" }}-v3 paths: - /home/circleci/go/pkg/mod test: docker: - - image: cimg/go:1.14 + - image: cimg/go:1.16 steps: - checkout - restore_cache: keys: - - go-mod-{{ checksum "go.sum" }}-v2 + - go-mod-{{ checksum "go.sum" }}-v3 - go-mod-{{ checksum "go.sum" }} - go-mod - run: command: make test workflows: - version: 2 build-and-test: jobs: - - build - test + - build: + requires: + - test diff --git a/Makefile b/Makefile index b303d52..6071f6c 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,9 @@ pi: make_build_dir small: make_build_dir go build -o build/bin/sliproad -ldflags="-s -w" - upx --brute build/bin/sliproad -9 --no-progress small_pi: make_build_dir env GOOS=linux GOARCH=arm GOARM=5 go build -o build/bin/sliproad-arm -ldflags="-s -w" - upx --brute build/bin/sliproad-arm -9 --no-progress run: go run webserver.go @@ -27,9 +25,8 @@ test: go test ./... -cover dist: clean make_build_dir small small_pi - cp -r assets/* build/assets - tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-arm.tar.gz build/assets build/bin/sliproad-arm README.md LICENSE - tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-x86.tar.gz build/assets build/bin/sliproad README.md LICENSE + tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-arm.tar.gz build/bin/sliproad-arm README.md LICENSE + tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-x86.tar.gz build/bin/sliproad README.md LICENSE clean: rm -rf build diff --git a/README.md b/README.md index b306d82..56ba1b1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,24 @@ -# sliproad -merging filesystems together +# Sliproad -## about +Merging filesystems together -this project aims to be an easy-to-user web API that allows the management of cloud storage, whether it be on -the host machine or part of a remote api. this is intended mostly for my own use, but i am documenting it in a way that -i hope allows others to pick it up and improve on it down the line. +## About -if something is unclear, feel free to open an issue :) +This project aims to be an easy-to-use web API and frontend that allows the +management of cloud storage alongside local filesystems. While this is intended +mostly for my own use, I am documenting it in a way that I hope allows others +to use it! -## configuration +## Configuration -unlike the initial version of this project, the current build uses _providers_ to determine how to handle various -functions related to files. currently, two are implemented, `disk` and `backblaze`, since they are the primary providers -i use myself. the providers you would like to use can be added to `providers.yml` alongside the binary. +Sliproad uses "Providers" to support various filesystems "types", whether it be +remote or local. Currently, three exist - `disk` for filesystems local to the +machine, `backblaze` to leverage Backblaze B2 file storage and `s3` for AWS S3 +(and other compatible providers). -for example, here is a sample configuration implementing both of them: +An example of leveraging all three, in various forms, can be found below. As +more are added, this example will be updated, and more examples can be found in +the `assets/config_examples` directory. ```yaml disk: @@ -24,53 +27,82 @@ disk: backblaze: provider: backblaze config: - appKeyId: APP_KEY_ID - appId: APP_ID - bucket: BUCKET_ID + bucket: some-bucket + applicationKeyId: application-key-id + applicationKey: application-key +s3: + provider: s3 + config: + region: eu-west-2 + bucket: some-bucket +# An example of an S3 compatible API, doesn't have to be Backblaze. +backblazes3: + provider: s3 + config: + bucket: some-bucket + region: us-west-000 + endpoint: s3.us-west-000.backblazeb2.com + keyid: key-id + keysecret: key-secret ``` -(read more here: [#providers](#providers)) +## Running -## running +After configuring the providers you would like to utilize, simply run +`./sliproad`. This will spin up the webserver at `127.0.0.1:3000`, listening on +all addresses. -after adding the providers you would like to use, the application can be run simply with `./nas`. it will attach to port -`:3000`. +## Frontend -## building +The frontend is a very lightweight JavaScript application and aims to be very +functional, if a bit rough around the edges. -this project uses go modules and a makefile, so building should be relatively straightforward. +![Screenshot_2021-05-29 Sliproad](https://user-images.githubusercontent.com/1878840/120085420-d63cbc80-c0cf-11eb-9fbb-b0b05a3f5d58.png) + +It should scale reasonably well for smaller devices. Because it's now bundled +into the binary (as opposed to distributed alongside), it's no longer possible +to swap it out for a custom frontend without serving the frontend seperately. + +## API + +This project is largely API-first, and documentation can be found here: + +https://github.com/gmemstr/sliproad/wiki/API + +## Building + +This project leverages a Makefile to macro common commands for running, testing +and building this project. - `make` will build the project for your system's architecture. - `make run` will run the project with `go run` - - `make pi` will build the project with the `GOOS=linux GOARCH=arm GOARM=5 go` flags set for raspberry pis. - -## providers + - `make pi` will build the project with the `GOOS=linux GOARCH=arm GOARM=5 go` flags set for Raspberry Pi. + - `make dist` will build and package the binaries for distribution. -"providers" provide a handful of functions to interact nicely with a filesystem, whether it be on local disk or on a -remote server via an api. best-effort is done to keep these performant, up to date and minimal. +### Adding Providers -there are a few built-in providers, and more can be added by opening a pull request. +New file providers can be implemented by building off the +`FileProviderInterface` struct, as the existing providers demonstrate. You can +then instruct the [`TranslateProvider()`](https://github.com/gmemstr/sliproad/blob/master/files/fileutils.go#L8-L21) function +that it exists and how to configure it. -|name|service|configuration example| -|----|-------|---------------------| -|disk|local filesystem|assets/config_examples/disk.yml| -|backblaze|backblaze b2|assets/config_examples/backblaze.yml| +## Authentication [!] -you can find a full configuration file under `assets/config_examples/providers.yml` +Authentication is a bit tricky and due to be reworked in the next iteration of +this project. Currently, support for [Keycloak](https://www.keycloak.org/) is +implemented, if a bit naively. You can turn this authentication requirement on +by adding `auth.yml` alongside your `providers.yml` file with the following: -### custom provider +```yaml +provider_url: "https://url-of-keycloak" +realm: "keycloak-realm" +redirect_base_url: "https://location-of-sliproad" +``` -custom file providers can be implemented by adding a new go file to the `files` module. it should -implement the `FileProviderInterface` interface. +Keycloak support is not currently actively supported, and is due to be removed +in the next major release of Sliproad. That said, if you encounter any major +bugs utilizing it before this, _please_ open an issue so I can dig in further. -## authentication +## Credits -basic authentication support utilizing [keycloak](https://keycloak.org/) has been implemented, but work -is being done to bring this more inline with the storage provider implementation. see `assets/config_examples/auth.yml` -for an example configuration - having this file alongside the binary will activate authentication on all -`/api/files` endpoints. note that this implementation is a work in progress, and a seperate branch will -contain further improvements. - -## icons - -SVG Icons provided by Paweł Kuna: https://github.com/tabler/tabler-icons (see assets/web/icons/README.md) \ No newline at end of file +SVG Icons provided by Paweł Kuna: https://github.com/tabler/tabler-icons (see assets/web/icons/README.md) diff --git a/assets/config_examples/README.md b/assets/config_examples/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/assets/config_examples/providers.yml b/assets/config_examples/providers.yml index 5c19b86..780b7c3 100644 --- a/assets/config_examples/providers.yml +++ b/assets/config_examples/providers.yml @@ -18,3 +18,16 @@ backblaze: applicationKeyId: aaaaaaaaaaaa applicationKey: aaaaaaaaaaaa bucket: aaaaaaaaaaaa +s3: + provider: s3 + config: + region: eu-west-2 + bucket: some-bucket +backblazes3: + provider: s3 + config: + bucket: sliproad-testing + region: us-west-000 + endpoint: s3.us-west-000.backblazeb2.com + keyid: key-id + keysecret: key-secret \ No newline at end of file diff --git a/assets/web/css/styles.css b/assets/web/css/styles.css index 611b274..c0449df 100644 --- a/assets/web/css/styles.css +++ b/assets/web/css/styles.css @@ -1,18 +1,31 @@ -:root { - /* https://coolors.co/fe4a49-fed766-009fb7-e6e6ea-f4f4f8 */ - --orange: rgba(254, 74, 73, 1); - --yellow: rgba(254, 215, 102, 1); - --blue: rgba(0, 159, 183, 1); - --platinum: rgba(230, 230, 234, 1); - --white: rgba(244, 244, 248, 1); +/* +Pink #F06DF2 +Black #02060D +Dark Blue #081226 +Blue #04ADBF +Yellow #F2E307 +*/ - --desktop-width: 1170px; +:root { + --black: #02060D; + --blue: #04ADBF; + --dark-blue: #081226; + --yellow: #F2E307; + --pink: #F06DF2; } body, h1 a { font-family: sans-serif; - color: var(--orange); - background-color: var(--white); + color: var(--black); + background-color: var(--black); +} + +.directory > span { + color: white; +} + +h1 a { + color: var(--pink); } .grid-lg { @@ -31,7 +44,7 @@ body, h1 a { font-weight: bold; text-decoration: none; transition: background-color 0.5s; - background-color: var(--blue); + background-color: var(--dark-blue); } .list { @@ -54,7 +67,7 @@ body, h1 a { position: relative; border-radius: 5px 0 0 5px; display: inline-block; - width: 93%; + width: 100%; } .list a img { @@ -64,42 +77,34 @@ body, h1 a { } .list a.file { - background-color: var(--orange); + background-color: transparent; + border: 1px solid var(--blue); + color: white; } .list a.directory { - background-color: var(--blue); + background-color: var(--dark-blue); } .grid-lg a:visited, .grid-lg a, .list a:visited, .list a { - color: var(--white); + color: white; } .grid-lg a:hover, .list a.directory:hover { - color: var(--blue); - background-color: var(--platinum); + background-color: var(--blue); transition: background-color 0.5s, color 0.5s; } .list a.file:hover { - color: var(--orange); - background-color: var(--platinum); + background-color: var(--dark-blue); transition: background-color 0.5s, color 0.5s; } -@media (prefers-color-scheme: dark) { - body, h1 a { background-color: black; } - .grid-lg a { - color: black; - } - .grid-lg a:visited, .grid-lg a { - color: black - } - .grid-lg a:hover { - color: lightgray; - background-color: darkgray; - transition: background-color 0.5s; - } +.directories { + display: grid; + gap: 2px 2px; + grid-template-columns: 1fr 1fr 1fr 1fr; + grid-template-rows: auto; } @media only screen and (max-width: 1170px) { @@ -127,12 +132,13 @@ input[type="file"] { } input[type="file"] + label, input[type="submit"], button { - color: var(--orange); + color: white; + background-color: transparent; padding: 10px; font-size: 1.25em; font-weight: 700; display: inline-block; - border: 2px solid var(--orange); + border: 2px solid var(--pink); border-radius: 5px; transition: background-color 0.5s, color 0.5s; } @@ -142,33 +148,33 @@ input[type="text"] { font-size: 1.25em; font-weight: 500; display: inline-block; - border: 2px solid var(--orange); + border: 2px solid var(--pink); border-radius: 5px; transition: background-color 0.5s, color 0.5s; width: 50%; + background-color: transparent; + color: white; } input[type="file"]:focus + label, input[type="file"] + label:hover, input[type="submit"]:hover, button:hover { - background-color: var(--orange); - color: var(--white); + color: white; transition: background-color 0.5s, color 0.5s; } progress { - background: var(--platinum); - border: 1px solid var(--orange); + border: 1px solid var(--pink); } progress::-webkit-progress-bar { - background: var(--orange); + background: var(--pink); } progress::-webkit-progress-value { - background: var(--orange); + background: var(--pink); } progress::-moz-progress-bar { - background: var(--orange); + background: var(--pink); } button { @@ -184,10 +190,19 @@ button { } .directory ~ button { - border-color: var(--blue); + border-color: var(--dark-blue); + background-color: white; } .directory ~ button:hover { - background-color: var(--blue); + background-color: grey; +} + +.file ~ button { + border-color: var(--blue); + background-color: white; +} +.file ~ button:hover { + background-color: grey; } .item, .forms { diff --git a/assets/web/javascript/app.js b/assets/web/javascript/app.js index 8f4d0ff..8423d26 100644 --- a/assets/web/javascript/app.js +++ b/assets/web/javascript/app.js @@ -19,6 +19,9 @@ function getFileListing(provider, path = "") { if (!files) { files = [] } + onlyfiles = files.filter(file => !file.IsDirectory) + onlydirs = files.filter(file => file.IsDirectory) + html`
@@ -30,10 +33,19 @@ function getFileListing(provider, path = "") {
-
- ${files.map(file => - `
- ${file.IsDirectory ? '' : ''}${file.Name} + + +
+ ${onlyfiles.map(file => + ` ` ).join('')} diff --git a/files/backblaze.go b/files/backblaze.go index 881ffb2..d1d2e21 100644 --- a/files/backblaze.go +++ b/files/backblaze.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "io/ioutil" + "mime" "net/http" + "path/filepath" "strings" ) @@ -138,11 +140,7 @@ func (bp *BackblazeProvider) GetDirectory(path string) Directory { return finalDir } -func (bp *BackblazeProvider) FilePath(path string) string { - return "" -} - -func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { +func (bp *BackblazeProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { client := &http.Client{} // Get bucket name >:( bucketIdPayload := fmt.Sprintf(`{"accountId": "%s", "bucketId": "%s"}`, bp.Name, bp.Bucket) @@ -150,20 +148,17 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { req, err := http.NewRequest("POST", bp.Location + "/b2api/v2/b2_list_buckets", bytes.NewBuffer([]byte(bucketIdPayload))) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } req.Header.Add("Authorization", bp.Authentication) res, err := client.Do(req) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } bucketData, err := ioutil.ReadAll(res.Body) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } var data BackblazeBucketInfoPayload @@ -175,22 +170,23 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) { url, bytes.NewBuffer([]byte(""))) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } req.Header.Add("Authorization", bp.Authentication) file, err := client.Do(req) if err != nil { - fmt.Println(err.Error()) - return + return nil, contenttype, err } - _, err = io.Copy(w, file.Body) - if err != nil { - fmt.Println(err.Error()) - return + contenttype = mime.TypeByExtension(filepath.Ext(path)) + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(file.Body, buf[:]) + contenttype = http.DetectContentType(buf[:n]) } + + return file.Body, contenttype, err } func (bp *BackblazeProvider) SaveFile(file io.Reader, filename string, path string) bool { diff --git a/files/disk.go b/files/disk.go index 3b3e048..a0c4b52 100644 --- a/files/disk.go +++ b/files/disk.go @@ -4,7 +4,10 @@ import ( "fmt" "io" "io/ioutil" + "mime" + "net/http" "os" + "path/filepath" "strings" ) @@ -42,13 +45,26 @@ func (dp *DiskProvider) GetDirectory(path string) Directory { } } -func (dp *DiskProvider) FilePath(path string) string { +func (dp *DiskProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { rp := strings.Join([]string{dp.Location,path}, "/") - return rp -} + f, err := os.Open(rp) + if err != nil { + return stream, contenttype, err + } -func (dp *DiskProvider) RemoteFile(path string, writer io.Writer) { - return + contenttype = mime.TypeByExtension(filepath.Ext(rp)) + + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(f, buf[:]) + contenttype = http.DetectContentType(buf[:n]) + _, err := f.Seek(0, io.SeekStart) + if err != nil { + return stream, contenttype, err + } + } + + return f, contenttype, nil } func (dp *DiskProvider) SaveFile(file io.Reader, filename string, path string) bool { diff --git a/files/fileprovider.go b/files/fileprovider.go index 94aea3d..f25da63 100644 --- a/files/fileprovider.go +++ b/files/fileprovider.go @@ -38,8 +38,7 @@ type FileInfo struct { type FileProviderInterface interface { Setup(args map[string]string) (ok bool) GetDirectory(path string) (directory Directory) - FilePath(path string) (realpath string) - RemoteFile(path string, writer io.Writer) + SendFile(path string) (stream io.Reader, contenttype string, err error) SaveFile(file io.Reader, filename string, path string) (ok bool) ObjectInfo(path string) (exists bool, isDir bool, location string) CreateDirectory(path string) (ok bool) @@ -58,13 +57,8 @@ func (f FileProvider) GetDirectory(path string) Directory { return Directory{} } -// FilePath returns the path to the file, whether it be a URL or local file path. -func (f FileProvider) FilePath(path string) string { - return "" -} - -// RemoteFile will bypass http.ServeContent() and instead write directly to the response. -func (f FileProvider) RemoteFile(path string, writer io.Writer) { +// SendFile returns a filestream, a valid MIME type for the file and an error. +func (f FileProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) { return } diff --git a/files/files_test.go b/files/files_test.go index 5ccacfd..2fdf8be 100644 --- a/files/files_test.go +++ b/files/files_test.go @@ -24,10 +24,6 @@ func TestFileProvider(t *testing.T) { t.Errorf("Default FileProvider GetDirectory() files returned %v, expected none.", getdirectory.Files) } - filepath := fp.FilePath(""); if filepath != "" { - t.Errorf("Default FileProvider FilePath() %v, expected nothing.", filepath) - } - savefile := fp.SaveFile(nil, "", ""); if savefile != false { t.Errorf("Default FileProvider SaveFile() attempted to save a file.") } @@ -87,10 +83,6 @@ func TestDiskProvider(t *testing.T) { t.Errorf("DiskProvider GetDirectory() files returned %v, expected 1.", getdirectory.Files) } - filepath := dp.FilePath("testing.txt"); if filepath != DISK_TESTING_GROUNDS + "/testing.txt"{ - t.Errorf("DiskProvider FilePath() returned %v, expected path.", filepath) - } - testfile := bytes.NewReader([]byte("second test file!")) savefile := dp.SaveFile(testfile, "second_test.txt", ""); if savefile != true { t.Errorf("DiskProvider SaveFile() could not save a file.") diff --git a/files/fileutils.go b/files/fileutils.go index f30a1ce..f13b1c2 100644 --- a/files/fileutils.go +++ b/files/fileutils.go @@ -17,6 +17,30 @@ func TranslateProvider(codename string, i *FileProviderInterface) { *i = bbProv return } + + if provider.Provider == "s3" { + s3Prov := &S3Provider{ + FileProvider: provider, + Region: provider.Config["region"], + Bucket: provider.Config["bucket"], + Endpoint: "", + KeyID: "", + KeySecret: "", + } + if _, ok := provider.Config["endpoint"]; ok { + s3Prov.Endpoint = provider.Config["endpoint"] + } + if _, ok := provider.Config["keyid"]; ok { + s3Prov.KeyID = provider.Config["keyid"] + } + if _, ok := provider.Config["keysecret"]; ok { + s3Prov.KeySecret = provider.Config["keysecret"] + } + + *i = s3Prov + return + } + *i = FileProvider{} } @@ -33,4 +57,4 @@ func SetupProviders() { fmt.Printf("%s initialized successfully\n", name) } } -} \ No newline at end of file +} diff --git a/files/s3.go b/files/s3.go new file mode 100644 index 0000000..ddd2516 --- /dev/null +++ b/files/s3.go @@ -0,0 +1,151 @@ +package files + +import ( + "fmt" + "io" + "mime" + "net/http" + "path/filepath" + + // I _really_ don't want to deal with AWS API stuff by hand. + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type S3Provider struct { + FileProvider + Region string + Bucket string + Endpoint string + KeyID string + KeySecret string + + svc s3.S3 + sess session.Session +} + +// Setup runs when the application starts up, and allows for things like authentication. +func (s *S3Provider) Setup(args map[string]string) bool { + config := &aws.Config{Region: aws.String(s.Region)} + if s.KeyID != "" && s.KeySecret != "" { + config = &aws.Config{ + Region: aws.String(s.Region), + Credentials: credentials.NewStaticCredentials(s.KeyID, s.KeySecret, ""), + } + } + if s.Endpoint != "" { + config.Endpoint = &s.Endpoint + } + ss, err := session.NewSession(config) + s.sess = *ss + if err != nil { + return false + } + s.svc = *s3.New(&s.sess) + return true +} + +// GetDirectory fetches a directory's contents. +func (s *S3Provider) GetDirectory(path string) Directory { + resp, err := s.svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.Bucket)}) + if err != nil { + fmt.Println(err) + return Directory{} + } + + dir := Directory{} + for _, item := range resp.Contents { + ik := *item.Key + // Why is this here? AWS returns a complete list of files, including + // files within subdirectories (prefixed with the dir name). So we can + // ignore directories altogether -- I would prefer to display them but + // not sure what the best method of distinguishing them in ObjectInfo() + // would be. + if ik[len(ik)-1:] == "/" { + continue + } + file := FileInfo{ + IsDirectory: false, + Name: *item.Key, + } + dir.Files = append(dir.Files, file) + } + + return dir +} + +func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string, err error) { + req, err := s.svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &path, + }) + if err != nil { + return stream, contenttype, err + } + + contenttype = mime.TypeByExtension(filepath.Ext(path)) + if contenttype == "" { + var buf [512]byte + n, _ := io.ReadFull(req.Body, buf[:]) + contenttype = http.DetectContentType(buf[:n]) + } + + return req.Body, contenttype, err +} + +func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool { + uploader := s3manager.NewUploader(&s.sess) + _, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: &s.Bucket, + Key: &filename, + Body: file, + }) + if err != nil { + return false + } + + return true +} + +func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) { + if path == "" { + return true, true, "" + } + + _, err := s.svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &path, + }) + if err != nil { + fmt.Println(err) + return false, false, "" + } + + return true, false, "" +} + +// CreateDirectory will create a directory on services that support it. +func (s *S3Provider) CreateDirectory(path string) bool { + return false +} + +// Delete simply deletes a file. This is expected to be a destructive action by default. +func (s *S3Provider) Delete(path string) bool { + _, err := s.svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(path)}) + if err != nil { + return false + } + + err = s.svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(s.Bucket), + Key: aws.String(path), + }) + if err != nil { + return false + } + + return true +} diff --git a/go.mod b/go.mod index a40e545..8e34ec3 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/gmemstr/nas -go 1.13 +go 1.16 require ( github.com/Nerzal/gocloak/v5 v5.1.0 + github.com/aws/aws-sdk-go v1.38.51 github.com/go-yaml/yaml v2.1.0+incompatible github.com/gorilla/mux v1.7.4 github.com/kr/pretty v0.2.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index 9094e9c..0e3f6cf 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Nerzal/gocloak/v5 v5.1.0 h1:1YP4+GoY1DZ1k7WyNNr8xbFyt55B9ORn2ZHu+XqUK0Q= github.com/Nerzal/gocloak/v5 v5.1.0/go.mod h1:8v53okuWiWXOKOS6qil8cOn7+5JSQfX1t1d+Nj8FpYk= +github.com/aws/aws-sdk-go v1.38.51 h1:aKQmbVbwOCuQSd8+fm/MR3bq0QOsu9Q7S+/QEND36oQ= +github.com/aws/aws-sdk-go v1.38.51/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -11,6 +13,10 @@ github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwn github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -24,10 +30,18 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/router/filerouter.go b/router/filerouter.go index f127f47..7b38c37 100644 --- a/router/filerouter.go +++ b/router/filerouter.go @@ -3,14 +3,14 @@ package router import ( "encoding/json" "fmt" - "github.com/gmemstr/nas/files" - "github.com/gorilla/mux" + "io" "net/http" "net/url" - "os" "sort" "strings" - "time" + + "github.com/gmemstr/nas/files" + "github.com/gorilla/mux" ) func handleProvider() handler { @@ -29,7 +29,7 @@ func handleProvider() handler { StatusCode: http.StatusInternalServerError, } } - ok, isDir, location := provider.ObjectInfo(filename) + ok, isDir, _ := provider.ObjectInfo(filename) if !ok { return &httpError{ Message: fmt.Sprintf("error locating file %s\n", filename), @@ -49,27 +49,25 @@ func handleProvider() handler { } w.Write(data) return nil - } + } else { + stream, contenttype, err := provider.SendFile(filename) - // If the file is local, attempt to use http.ServeContent for correct headers. - if location == files.FileIsLocal { - rp := provider.FilePath(filename) - if rp != "" { - f, err := os.Open(rp) - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error opening file %s\n", rp), - StatusCode: http.StatusInternalServerError, - } + if err != nil { + return &httpError{ + Message: fmt.Sprintf("error finding file %s\n", vars["file"]), + StatusCode: http.StatusNotFound, + } + } + w.Header().Set("Content-Type", contenttype) + _, err = io.Copy(w, stream) + if err != nil { + return &httpError{ + Message: fmt.Sprintf("unable to write %s\n", vars["file"]), + StatusCode: http.StatusBadGateway, } - http.ServeContent(w, r, filename, time.Time{}, f) } } - // If the file is remote, then delegate the writing to the response to the provider. - // This isn't a great workaround, but avoids caching the whole file in mem or on disk. - if location == files.FileIsRemote { - provider.RemoteFile(filename, w) - } + return nil } diff --git a/router/router.go b/router/router.go index 3ad6d88..bbf4958 100644 --- a/router/router.go +++ b/router/router.go @@ -1,12 +1,9 @@ package router import ( - "fmt" - "io" "log" "net/http" - "os" - "strconv" + "io/fs" "github.com/gorilla/mux" ) @@ -37,21 +34,10 @@ func handle(handlers ...handler) http.Handler { } // Init initializes the main router and all routes for the application. -func Init() *mux.Router { +func Init(sc fs.FS) *mux.Router { r := mux.NewRouter() - // "Static" paths - r.PathPrefix("/javascript/").Handler(http.StripPrefix("/javascript/", http.FileServer(http.Dir("assets/web/javascript")))) - r.PathPrefix("/css/").Handler(http.StripPrefix("/css/", http.FileServer(http.Dir("assets/web/css")))) - r.PathPrefix("/icons/").Handler(http.StripPrefix("/icons/", http.FileServer(http.Dir("assets/web/icons")))) - - // Paths that require specific handlers - r.Handle("/", handle( - requiresAuth(), - rootHandler(), - )).Methods("GET") - // File & Provider API r.Handle("/api/providers", handle( requiresAuth(), @@ -61,7 +47,7 @@ func Init() *mux.Router { r.Handle(`/api/files/{provider:[a-zA-Z0-9]+\/*}`, handle( requiresAuth(), handleProvider(), - )).Methods("GET", "POST") + )).Methods("GET", "POST", "DELETE") r.Handle(`/api/files/{provider:[a-zA-Z0-9]+}/{file:.+}`, handle( requiresAuth(), @@ -73,38 +59,7 @@ func Init() *mux.Router { callbackAuth(), )).Methods("GET", "POST") + r.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.FS(sc)))) + return r } - -// Handles serving index page. -func rootHandler() handler { - return func(context *requestContext, w http.ResponseWriter, r *http.Request) *httpError { - f, err := os.Open("assets/web/index.html") - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error serving index page from assets/web"), - StatusCode: http.StatusInternalServerError, - } - } - - defer f.Close() - stats, err := f.Stat() - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error serving index page from assets/web"), - StatusCode: http.StatusInternalServerError, - } - } else { - w.Header().Add("Content-Length", strconv.FormatInt(stats.Size(), 10)) - } - - _, err = io.Copy(w, f) - if err != nil { - return &httpError{ - Message: fmt.Sprintf("error serving index page from assets/web"), - StatusCode: http.StatusInternalServerError, - } - } - return nil - } -} diff --git a/webserver.go b/webserver.go index 978bee6..613bf35 100644 --- a/webserver.go +++ b/webserver.go @@ -1,7 +1,9 @@ package main import ( + "embed" "fmt" + "io/fs" "io/ioutil" "log" "net/http" @@ -12,6 +14,9 @@ import ( "github.com/go-yaml/yaml" ) +//go:embed assets/web/* +var sc embed.FS + // Main function that defines routes func main() { // Initialize file providers. @@ -39,7 +44,8 @@ func main() { fmt.Println("Keycloak configured") } - r := router.Init() + fsys, err := fs.Sub(sc, "assets/web") + r := router.Init(fsys) fmt.Println("Your sliproad instance is live on port :3000") log.Fatal(http.ListenAndServe(":3000", r)) }