Merge pull request #15 from gmemstr/staging

This commit is contained in:
Gabriel Simmer 2021-05-29 22:56:04 +01:00 committed by GitHub
commit 17e489ad1c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 440 additions and 226 deletions

View file

@ -1,41 +1,40 @@
version: 2.1
orbs:
upx: circleci/upx@1.0.1
jobs:
build:
docker:
- image: cimg/go:1.14
- image: cimg/go:1.16
steps:
- checkout
- restore_cache:
keys:
- go-mod-{{ checksum "go.sum" }}-v2
- go-mod-{{ checksum "go.sum" }}-v3
- go-mod-{{ checksum "go.sum" }}
- go-mod
- upx/install
- run:
command: make dist
- store_artifacts:
path: build
- save_cache:
key: go-mod-{{ checksum "go.sum" }}-v2
key: go-mod-{{ checksum "go.sum" }}-v3
paths:
- /home/circleci/go/pkg/mod
test:
docker:
- image: cimg/go:1.14
- image: cimg/go:1.16
steps:
- checkout
- restore_cache:
keys:
- go-mod-{{ checksum "go.sum" }}-v2
- go-mod-{{ checksum "go.sum" }}-v3
- go-mod-{{ checksum "go.sum" }}
- go-mod
- run:
command: make test
workflows:
version: 2
build-and-test:
jobs:
- build
- test
- build:
requires:
- test

View file

@ -14,11 +14,9 @@ pi: make_build_dir
small: make_build_dir
go build -o build/bin/sliproad -ldflags="-s -w"
upx --brute build/bin/sliproad -9 --no-progress
small_pi: make_build_dir
env GOOS=linux GOARCH=arm GOARM=5 go build -o build/bin/sliproad-arm -ldflags="-s -w"
upx --brute build/bin/sliproad-arm -9 --no-progress
run:
go run webserver.go
@ -27,9 +25,8 @@ test:
go test ./... -cover
dist: clean make_build_dir small small_pi
cp -r assets/* build/assets
tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-arm.tar.gz build/assets build/bin/sliproad-arm README.md LICENSE
tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-x86.tar.gz build/assets build/bin/sliproad README.md LICENSE
tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-arm.tar.gz build/bin/sliproad-arm README.md LICENSE
tar -czf build/tars/sliproad-$(SLIPROAD_VERSION)-x86.tar.gz build/bin/sliproad README.md LICENSE
clean:
rm -rf build

122
README.md
View file

@ -1,21 +1,24 @@
# sliproad
merging filesystems together
# Sliproad
## about
Merging filesystems together
this project aims to be an easy-to-user web API that allows the management of cloud storage, whether it be on
the host machine or part of a remote api. this is intended mostly for my own use, but i am documenting it in a way that
i hope allows others to pick it up and improve on it down the line.
## About
if something is unclear, feel free to open an issue :)
This project aims to be an easy-to-use web API and frontend that allows the
management of cloud storage alongside local filesystems. While this is intended
mostly for my own use, I am documenting it in a way that I hope allows others
to use it!
## configuration
## Configuration
unlike the initial version of this project, the current build uses _providers_ to determine how to handle various
functions related to files. currently, two are implemented, `disk` and `backblaze`, since they are the primary providers
i use myself. the providers you would like to use can be added to `providers.yml` alongside the binary.
Sliproad uses "Providers" to support various filesystems "types", whether it be
remote or local. Currently, three exist - `disk` for filesystems local to the
machine, `backblaze` to leverage Backblaze B2 file storage and `s3` for AWS S3
(and other compatible providers).
for example, here is a sample configuration implementing both of them:
An example of leveraging all three, in various forms, can be found below. As
more are added, this example will be updated, and more examples can be found in
the `assets/config_examples` directory.
```yaml
disk:
@ -24,53 +27,82 @@ disk:
backblaze:
provider: backblaze
config:
appKeyId: APP_KEY_ID
appId: APP_ID
bucket: BUCKET_ID
bucket: some-bucket
applicationKeyId: application-key-id
applicationKey: application-key
s3:
provider: s3
config:
region: eu-west-2
bucket: some-bucket
# An example of an S3 compatible API, doesn't have to be Backblaze.
backblazes3:
provider: s3
config:
bucket: some-bucket
region: us-west-000
endpoint: s3.us-west-000.backblazeb2.com
keyid: key-id
keysecret: key-secret
```
(read more here: [#providers](#providers))
## Running
## running
After configuring the providers you would like to utilize, simply run
`./sliproad`. This will spin up the webserver at `127.0.0.1:3000`, listening on
all addresses.
after adding the providers you would like to use, the application can be run simply with `./nas`. it will attach to port
`:3000`.
## Frontend
## building
The frontend is a very lightweight JavaScript application and aims to be very
functional, if a bit rough around the edges.
this project uses go modules and a makefile, so building should be relatively straightforward.
![Screenshot_2021-05-29 Sliproad](https://user-images.githubusercontent.com/1878840/120085420-d63cbc80-c0cf-11eb-9fbb-b0b05a3f5d58.png)
It should scale reasonably well for smaller devices. Because it's now bundled
into the binary (as opposed to distributed alongside), it's no longer possible
to swap it out for a custom frontend without serving the frontend seperately.
## API
This project is largely API-first, and documentation can be found here:
https://github.com/gmemstr/sliproad/wiki/API
## Building
This project leverages a Makefile to macro common commands for running, testing
and building this project.
- `make` will build the project for your system's architecture.
- `make run` will run the project with `go run`
- `make pi` will build the project with the `GOOS=linux GOARCH=arm GOARM=5 go` flags set for raspberry pis.
## providers
- `make pi` will build the project with the `GOOS=linux GOARCH=arm GOARM=5 go` flags set for Raspberry Pi.
- `make dist` will build and package the binaries for distribution.
"providers" provide a handful of functions to interact nicely with a filesystem, whether it be on local disk or on a
remote server via an api. best-effort is done to keep these performant, up to date and minimal.
### Adding Providers
there are a few built-in providers, and more can be added by opening a pull request.
New file providers can be implemented by building off the
`FileProviderInterface` struct, as the existing providers demonstrate. You can
then instruct the [`TranslateProvider()`](https://github.com/gmemstr/sliproad/blob/master/files/fileutils.go#L8-L21) function
that it exists and how to configure it.
|name|service|configuration example|
|----|-------|---------------------|
|disk|local filesystem|assets/config_examples/disk.yml|
|backblaze|backblaze b2|assets/config_examples/backblaze.yml|
## Authentication [!]
you can find a full configuration file under `assets/config_examples/providers.yml`
Authentication is a bit tricky and due to be reworked in the next iteration of
this project. Currently, support for [Keycloak](https://www.keycloak.org/) is
implemented, if a bit naively. You can turn this authentication requirement on
by adding `auth.yml` alongside your `providers.yml` file with the following:
### custom provider
```yaml
provider_url: "https://url-of-keycloak"
realm: "keycloak-realm"
redirect_base_url: "https://location-of-sliproad"
```
custom file providers can be implemented by adding a new go file to the `files` module. it should
implement the `FileProviderInterface` interface.
Keycloak support is not currently actively supported, and is due to be removed
in the next major release of Sliproad. That said, if you encounter any major
bugs utilizing it before this, _please_ open an issue so I can dig in further.
## authentication
## Credits
basic authentication support utilizing [keycloak](https://keycloak.org/) has been implemented, but work
is being done to bring this more inline with the storage provider implementation. see `assets/config_examples/auth.yml`
for an example configuration - having this file alongside the binary will activate authentication on all
`/api/files` endpoints. note that this implementation is a work in progress, and a seperate branch will
contain further improvements.
## icons
SVG Icons provided by Paweł Kuna: https://github.com/tabler/tabler-icons (see assets/web/icons/README.md)
SVG Icons provided by Paweł Kuna: https://github.com/tabler/tabler-icons (see assets/web/icons/README.md)

View file

@ -18,3 +18,16 @@ backblaze:
applicationKeyId: aaaaaaaaaaaa
applicationKey: aaaaaaaaaaaa
bucket: aaaaaaaaaaaa
s3:
provider: s3
config:
region: eu-west-2
bucket: some-bucket
backblazes3:
provider: s3
config:
bucket: sliproad-testing
region: us-west-000
endpoint: s3.us-west-000.backblazeb2.com
keyid: key-id
keysecret: key-secret

View file

@ -1,18 +1,31 @@
:root {
/* https://coolors.co/fe4a49-fed766-009fb7-e6e6ea-f4f4f8 */
--orange: rgba(254, 74, 73, 1);
--yellow: rgba(254, 215, 102, 1);
--blue: rgba(0, 159, 183, 1);
--platinum: rgba(230, 230, 234, 1);
--white: rgba(244, 244, 248, 1);
/*
Pink #F06DF2
Black #02060D
Dark Blue #081226
Blue #04ADBF
Yellow #F2E307
*/
--desktop-width: 1170px;
:root {
--black: #02060D;
--blue: #04ADBF;
--dark-blue: #081226;
--yellow: #F2E307;
--pink: #F06DF2;
}
body, h1 a {
font-family: sans-serif;
color: var(--orange);
background-color: var(--white);
color: var(--black);
background-color: var(--black);
}
.directory > span {
color: white;
}
h1 a {
color: var(--pink);
}
.grid-lg {
@ -31,7 +44,7 @@ body, h1 a {
font-weight: bold;
text-decoration: none;
transition: background-color 0.5s;
background-color: var(--blue);
background-color: var(--dark-blue);
}
.list {
@ -54,7 +67,7 @@ body, h1 a {
position: relative;
border-radius: 5px 0 0 5px;
display: inline-block;
width: 93%;
width: 100%;
}
.list a img {
@ -64,42 +77,34 @@ body, h1 a {
}
.list a.file {
background-color: var(--orange);
background-color: transparent;
border: 1px solid var(--blue);
color: white;
}
.list a.directory {
background-color: var(--blue);
background-color: var(--dark-blue);
}
.grid-lg a:visited, .grid-lg a,
.list a:visited, .list a {
color: var(--white);
color: white;
}
.grid-lg a:hover,
.list a.directory:hover {
color: var(--blue);
background-color: var(--platinum);
background-color: var(--blue);
transition: background-color 0.5s, color 0.5s;
}
.list a.file:hover {
color: var(--orange);
background-color: var(--platinum);
background-color: var(--dark-blue);
transition: background-color 0.5s, color 0.5s;
}
@media (prefers-color-scheme: dark) {
body, h1 a { background-color: black; }
.grid-lg a {
color: black;
}
.grid-lg a:visited, .grid-lg a {
color: black
}
.grid-lg a:hover {
color: lightgray;
background-color: darkgray;
transition: background-color 0.5s;
}
.directories {
display: grid;
gap: 2px 2px;
grid-template-columns: 1fr 1fr 1fr 1fr;
grid-template-rows: auto;
}
@media only screen and (max-width: 1170px) {
@ -127,12 +132,13 @@ input[type="file"] {
}
input[type="file"] + label, input[type="submit"], button {
color: var(--orange);
color: white;
background-color: transparent;
padding: 10px;
font-size: 1.25em;
font-weight: 700;
display: inline-block;
border: 2px solid var(--orange);
border: 2px solid var(--pink);
border-radius: 5px;
transition: background-color 0.5s, color 0.5s;
}
@ -142,33 +148,33 @@ input[type="text"] {
font-size: 1.25em;
font-weight: 500;
display: inline-block;
border: 2px solid var(--orange);
border: 2px solid var(--pink);
border-radius: 5px;
transition: background-color 0.5s, color 0.5s;
width: 50%;
background-color: transparent;
color: white;
}
input[type="file"]:focus + label,
input[type="file"] + label:hover,
input[type="submit"]:hover,
button:hover {
background-color: var(--orange);
color: var(--white);
color: white;
transition: background-color 0.5s, color 0.5s;
}
progress {
background: var(--platinum);
border: 1px solid var(--orange);
border: 1px solid var(--pink);
}
progress::-webkit-progress-bar {
background: var(--orange);
background: var(--pink);
}
progress::-webkit-progress-value {
background: var(--orange);
background: var(--pink);
}
progress::-moz-progress-bar {
background: var(--orange);
background: var(--pink);
}
button {
@ -184,10 +190,19 @@ button {
}
.directory ~ button {
border-color: var(--blue);
border-color: var(--dark-blue);
background-color: white;
}
.directory ~ button:hover {
background-color: var(--blue);
background-color: grey;
}
.file ~ button {
border-color: var(--blue);
background-color: white;
}
.file ~ button:hover {
background-color: grey;
}
.item, .forms {

View file

@ -19,6 +19,9 @@ function getFileListing(provider, path = "") {
if (!files) {
files = []
}
onlyfiles = files.filter(file => !file.IsDirectory)
onlydirs = files.filter(file => file.IsDirectory)
html`
<div class="forms">
<form id="uploadfile" action="#" method="post">
@ -30,10 +33,19 @@ function getFileListing(provider, path = "") {
<input type="submit" value="Create Directory" id="newdir_submit">
</form>
</div>
<div class="list">
${files.map(file =>
`<div class="item"><a class="${file.IsDirectory ? "directory" : "file"}" href="${!file.IsDirectory ? `/api/files/${provider}${path}/${file.Name}` : `#${provider}${path === "" ? "" : path}/${file.Name}`}">
<span>${file.IsDirectory ? '<img src="/icons/folder.svg"/>' : '<img src="/icons/file.svg"/>'}${file.Name}</span>
<div class="directories list">
${onlydirs.map(directory =>
`<div class="item"><a class="directory" href="#${provider}${path === "" ? "" : path}/${directory.Name}">
<span>${directory.Name}/</span>
</a><button onclick="deleteFile('${provider}', '${path === "" ? '' : path}', '${directory.Name}')"><img src="/icons/trash.svg"/></button></div>
`
).join('')}
</div>
<div class="files list">
${onlyfiles.map(file =>
`<div class="item"><a class="file" href="/api/files/${provider}${path}/${file.Name}" target="__blank">
<span>${file.Name}</span>
</a><button onclick="deleteFile('${provider}', '${path === "" ? '' : path}', '${file.Name}')"><img src="/icons/trash.svg"/></button></div>
`
).join('')}

View file

@ -7,7 +7,9 @@ import (
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"path/filepath"
"strings"
)
@ -138,11 +140,7 @@ func (bp *BackblazeProvider) GetDirectory(path string) Directory {
return finalDir
}
func (bp *BackblazeProvider) FilePath(path string) string {
return ""
}
func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) {
func (bp *BackblazeProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) {
client := &http.Client{}
// Get bucket name >:(
bucketIdPayload := fmt.Sprintf(`{"accountId": "%s", "bucketId": "%s"}`, bp.Name, bp.Bucket)
@ -150,20 +148,17 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) {
req, err := http.NewRequest("POST", bp.Location + "/b2api/v2/b2_list_buckets",
bytes.NewBuffer([]byte(bucketIdPayload)))
if err != nil {
fmt.Println(err.Error())
return
return nil, contenttype, err
}
req.Header.Add("Authorization", bp.Authentication)
res, err := client.Do(req)
if err != nil {
fmt.Println(err.Error())
return
return nil, contenttype, err
}
bucketData, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err.Error())
return
return nil, contenttype, err
}
var data BackblazeBucketInfoPayload
@ -175,22 +170,23 @@ func (bp *BackblazeProvider) RemoteFile(path string, w io.Writer) {
url,
bytes.NewBuffer([]byte("")))
if err != nil {
fmt.Println(err.Error())
return
return nil, contenttype, err
}
req.Header.Add("Authorization", bp.Authentication)
file, err := client.Do(req)
if err != nil {
fmt.Println(err.Error())
return
return nil, contenttype, err
}
_, err = io.Copy(w, file.Body)
if err != nil {
fmt.Println(err.Error())
return
contenttype = mime.TypeByExtension(filepath.Ext(path))
if contenttype == "" {
var buf [512]byte
n, _ := io.ReadFull(file.Body, buf[:])
contenttype = http.DetectContentType(buf[:n])
}
return file.Body, contenttype, err
}
func (bp *BackblazeProvider) SaveFile(file io.Reader, filename string, path string) bool {

View file

@ -4,7 +4,10 @@ import (
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
)
@ -42,13 +45,26 @@ func (dp *DiskProvider) GetDirectory(path string) Directory {
}
}
func (dp *DiskProvider) FilePath(path string) string {
func (dp *DiskProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) {
rp := strings.Join([]string{dp.Location,path}, "/")
return rp
}
f, err := os.Open(rp)
if err != nil {
return stream, contenttype, err
}
func (dp *DiskProvider) RemoteFile(path string, writer io.Writer) {
return
contenttype = mime.TypeByExtension(filepath.Ext(rp))
if contenttype == "" {
var buf [512]byte
n, _ := io.ReadFull(f, buf[:])
contenttype = http.DetectContentType(buf[:n])
_, err := f.Seek(0, io.SeekStart)
if err != nil {
return stream, contenttype, err
}
}
return f, contenttype, nil
}
func (dp *DiskProvider) SaveFile(file io.Reader, filename string, path string) bool {

View file

@ -38,8 +38,7 @@ type FileInfo struct {
type FileProviderInterface interface {
Setup(args map[string]string) (ok bool)
GetDirectory(path string) (directory Directory)
FilePath(path string) (realpath string)
RemoteFile(path string, writer io.Writer)
SendFile(path string) (stream io.Reader, contenttype string, err error)
SaveFile(file io.Reader, filename string, path string) (ok bool)
ObjectInfo(path string) (exists bool, isDir bool, location string)
CreateDirectory(path string) (ok bool)
@ -58,13 +57,8 @@ func (f FileProvider) GetDirectory(path string) Directory {
return Directory{}
}
// FilePath returns the path to the file, whether it be a URL or local file path.
func (f FileProvider) FilePath(path string) string {
return ""
}
// RemoteFile will bypass http.ServeContent() and instead write directly to the response.
func (f FileProvider) RemoteFile(path string, writer io.Writer) {
// SendFile returns a filestream, a valid MIME type for the file and an error.
func (f FileProvider) SendFile(path string) (stream io.Reader, contenttype string, err error) {
return
}

View file

@ -24,10 +24,6 @@ func TestFileProvider(t *testing.T) {
t.Errorf("Default FileProvider GetDirectory() files returned %v, expected none.", getdirectory.Files)
}
filepath := fp.FilePath(""); if filepath != "" {
t.Errorf("Default FileProvider FilePath() %v, expected nothing.", filepath)
}
savefile := fp.SaveFile(nil, "", ""); if savefile != false {
t.Errorf("Default FileProvider SaveFile() attempted to save a file.")
}
@ -87,10 +83,6 @@ func TestDiskProvider(t *testing.T) {
t.Errorf("DiskProvider GetDirectory() files returned %v, expected 1.", getdirectory.Files)
}
filepath := dp.FilePath("testing.txt"); if filepath != DISK_TESTING_GROUNDS + "/testing.txt"{
t.Errorf("DiskProvider FilePath() returned %v, expected path.", filepath)
}
testfile := bytes.NewReader([]byte("second test file!"))
savefile := dp.SaveFile(testfile, "second_test.txt", ""); if savefile != true {
t.Errorf("DiskProvider SaveFile() could not save a file.")

View file

@ -17,6 +17,30 @@ func TranslateProvider(codename string, i *FileProviderInterface) {
*i = bbProv
return
}
if provider.Provider == "s3" {
s3Prov := &S3Provider{
FileProvider: provider,
Region: provider.Config["region"],
Bucket: provider.Config["bucket"],
Endpoint: "",
KeyID: "",
KeySecret: "",
}
if _, ok := provider.Config["endpoint"]; ok {
s3Prov.Endpoint = provider.Config["endpoint"]
}
if _, ok := provider.Config["keyid"]; ok {
s3Prov.KeyID = provider.Config["keyid"]
}
if _, ok := provider.Config["keysecret"]; ok {
s3Prov.KeySecret = provider.Config["keysecret"]
}
*i = s3Prov
return
}
*i = FileProvider{}
}
@ -33,4 +57,4 @@ func SetupProviders() {
fmt.Printf("%s initialized successfully\n", name)
}
}
}
}

151
files/s3.go Normal file
View file

@ -0,0 +1,151 @@
package files
import (
"fmt"
"io"
"mime"
"net/http"
"path/filepath"
// I _really_ don't want to deal with AWS API stuff by hand.
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
type S3Provider struct {
FileProvider
Region string
Bucket string
Endpoint string
KeyID string
KeySecret string
svc s3.S3
sess session.Session
}
// Setup runs when the application starts up, and allows for things like authentication.
func (s *S3Provider) Setup(args map[string]string) bool {
config := &aws.Config{Region: aws.String(s.Region)}
if s.KeyID != "" && s.KeySecret != "" {
config = &aws.Config{
Region: aws.String(s.Region),
Credentials: credentials.NewStaticCredentials(s.KeyID, s.KeySecret, ""),
}
}
if s.Endpoint != "" {
config.Endpoint = &s.Endpoint
}
ss, err := session.NewSession(config)
s.sess = *ss
if err != nil {
return false
}
s.svc = *s3.New(&s.sess)
return true
}
// GetDirectory fetches a directory's contents.
func (s *S3Provider) GetDirectory(path string) Directory {
resp, err := s.svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.Bucket)})
if err != nil {
fmt.Println(err)
return Directory{}
}
dir := Directory{}
for _, item := range resp.Contents {
ik := *item.Key
// Why is this here? AWS returns a complete list of files, including
// files within subdirectories (prefixed with the dir name). So we can
// ignore directories altogether -- I would prefer to display them but
// not sure what the best method of distinguishing them in ObjectInfo()
// would be.
if ik[len(ik)-1:] == "/" {
continue
}
file := FileInfo{
IsDirectory: false,
Name: *item.Key,
}
dir.Files = append(dir.Files, file)
}
return dir
}
func (s *S3Provider) SendFile(path string) (stream io.Reader, contenttype string, err error) {
req, err := s.svc.GetObject(&s3.GetObjectInput{
Bucket: &s.Bucket,
Key: &path,
})
if err != nil {
return stream, contenttype, err
}
contenttype = mime.TypeByExtension(filepath.Ext(path))
if contenttype == "" {
var buf [512]byte
n, _ := io.ReadFull(req.Body, buf[:])
contenttype = http.DetectContentType(buf[:n])
}
return req.Body, contenttype, err
}
func (s *S3Provider) SaveFile(file io.Reader, filename string, path string) bool {
uploader := s3manager.NewUploader(&s.sess)
_, err := uploader.Upload(&s3manager.UploadInput{
Bucket: &s.Bucket,
Key: &filename,
Body: file,
})
if err != nil {
return false
}
return true
}
func (s *S3Provider) ObjectInfo(path string) (bool, bool, string) {
if path == "" {
return true, true, ""
}
_, err := s.svc.GetObject(&s3.GetObjectInput{
Bucket: &s.Bucket,
Key: &path,
})
if err != nil {
fmt.Println(err)
return false, false, ""
}
return true, false, ""
}
// CreateDirectory will create a directory on services that support it.
func (s *S3Provider) CreateDirectory(path string) bool {
return false
}
// Delete simply deletes a file. This is expected to be a destructive action by default.
func (s *S3Provider) Delete(path string) bool {
_, err := s.svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(path)})
if err != nil {
return false
}
err = s.svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(path),
})
if err != nil {
return false
}
return true
}

4
go.mod
View file

@ -1,12 +1,12 @@
module github.com/gmemstr/nas
go 1.13
go 1.16
require (
github.com/Nerzal/gocloak/v5 v5.1.0
github.com/aws/aws-sdk-go v1.38.51
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/gorilla/mux v1.7.4
github.com/kr/pretty v0.2.0 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
)

16
go.sum
View file

@ -1,5 +1,7 @@
github.com/Nerzal/gocloak/v5 v5.1.0 h1:1YP4+GoY1DZ1k7WyNNr8xbFyt55B9ORn2ZHu+XqUK0Q=
github.com/Nerzal/gocloak/v5 v5.1.0/go.mod h1:8v53okuWiWXOKOS6qil8cOn7+5JSQfX1t1d+Nj8FpYk=
github.com/aws/aws-sdk-go v1.38.51 h1:aKQmbVbwOCuQSd8+fm/MR3bq0QOsu9Q7S+/QEND36oQ=
github.com/aws/aws-sdk-go v1.38.51/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -11,6 +13,10 @@ github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwn
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -24,10 +30,18 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View file

@ -3,14 +3,14 @@ package router
import (
"encoding/json"
"fmt"
"github.com/gmemstr/nas/files"
"github.com/gorilla/mux"
"io"
"net/http"
"net/url"
"os"
"sort"
"strings"
"time"
"github.com/gmemstr/nas/files"
"github.com/gorilla/mux"
)
func handleProvider() handler {
@ -29,7 +29,7 @@ func handleProvider() handler {
StatusCode: http.StatusInternalServerError,
}
}
ok, isDir, location := provider.ObjectInfo(filename)
ok, isDir, _ := provider.ObjectInfo(filename)
if !ok {
return &httpError{
Message: fmt.Sprintf("error locating file %s\n", filename),
@ -49,27 +49,25 @@ func handleProvider() handler {
}
w.Write(data)
return nil
}
} else {
stream, contenttype, err := provider.SendFile(filename)
// If the file is local, attempt to use http.ServeContent for correct headers.
if location == files.FileIsLocal {
rp := provider.FilePath(filename)
if rp != "" {
f, err := os.Open(rp)
if err != nil {
return &httpError{
Message: fmt.Sprintf("error opening file %s\n", rp),
StatusCode: http.StatusInternalServerError,
}
if err != nil {
return &httpError{
Message: fmt.Sprintf("error finding file %s\n", vars["file"]),
StatusCode: http.StatusNotFound,
}
}
w.Header().Set("Content-Type", contenttype)
_, err = io.Copy(w, stream)
if err != nil {
return &httpError{
Message: fmt.Sprintf("unable to write %s\n", vars["file"]),
StatusCode: http.StatusBadGateway,
}
http.ServeContent(w, r, filename, time.Time{}, f)
}
}
// If the file is remote, then delegate the writing to the response to the provider.
// This isn't a great workaround, but avoids caching the whole file in mem or on disk.
if location == files.FileIsRemote {
provider.RemoteFile(filename, w)
}
return nil
}

View file

@ -1,12 +1,9 @@
package router
import (
"fmt"
"io"
"log"
"net/http"
"os"
"strconv"
"io/fs"
"github.com/gorilla/mux"
)
@ -37,21 +34,10 @@ func handle(handlers ...handler) http.Handler {
}
// Init initializes the main router and all routes for the application.
func Init() *mux.Router {
func Init(sc fs.FS) *mux.Router {
r := mux.NewRouter()
// "Static" paths
r.PathPrefix("/javascript/").Handler(http.StripPrefix("/javascript/", http.FileServer(http.Dir("assets/web/javascript"))))
r.PathPrefix("/css/").Handler(http.StripPrefix("/css/", http.FileServer(http.Dir("assets/web/css"))))
r.PathPrefix("/icons/").Handler(http.StripPrefix("/icons/", http.FileServer(http.Dir("assets/web/icons"))))
// Paths that require specific handlers
r.Handle("/", handle(
requiresAuth(),
rootHandler(),
)).Methods("GET")
// File & Provider API
r.Handle("/api/providers", handle(
requiresAuth(),
@ -61,7 +47,7 @@ func Init() *mux.Router {
r.Handle(`/api/files/{provider:[a-zA-Z0-9]+\/*}`, handle(
requiresAuth(),
handleProvider(),
)).Methods("GET", "POST")
)).Methods("GET", "POST", "DELETE")
r.Handle(`/api/files/{provider:[a-zA-Z0-9]+}/{file:.+}`, handle(
requiresAuth(),
@ -73,38 +59,7 @@ func Init() *mux.Router {
callbackAuth(),
)).Methods("GET", "POST")
r.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.FS(sc))))
return r
}
// Handles serving index page.
func rootHandler() handler {
return func(context *requestContext, w http.ResponseWriter, r *http.Request) *httpError {
f, err := os.Open("assets/web/index.html")
if err != nil {
return &httpError{
Message: fmt.Sprintf("error serving index page from assets/web"),
StatusCode: http.StatusInternalServerError,
}
}
defer f.Close()
stats, err := f.Stat()
if err != nil {
return &httpError{
Message: fmt.Sprintf("error serving index page from assets/web"),
StatusCode: http.StatusInternalServerError,
}
} else {
w.Header().Add("Content-Length", strconv.FormatInt(stats.Size(), 10))
}
_, err = io.Copy(w, f)
if err != nil {
return &httpError{
Message: fmt.Sprintf("error serving index page from assets/web"),
StatusCode: http.StatusInternalServerError,
}
}
return nil
}
}

View file

@ -1,7 +1,9 @@
package main
import (
"embed"
"fmt"
"io/fs"
"io/ioutil"
"log"
"net/http"
@ -12,6 +14,9 @@ import (
"github.com/go-yaml/yaml"
)
//go:embed assets/web/*
var sc embed.FS
// Main function that defines routes
func main() {
// Initialize file providers.
@ -39,7 +44,8 @@ func main() {
fmt.Println("Keycloak configured")
}
r := router.Init()
fsys, err := fs.Sub(sc, "assets/web")
r := router.Init(fsys)
fmt.Println("Your sliproad instance is live on port :3000")
log.Fatal(http.ListenAndServe(":3000", r))
}