niklas/s3browser-backend
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

MORE CACHES

Browse Source
This commit is contained in:
Djeeberjr 2021-08-03 23:10:23 +02:00
parent 600547342a
commit 6598088024
3 changed files with 137 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
internal/dataloader.go
View File

@ -8,6 +8,7 @@ import (
"github.com/graph-gophers/dataloader"
"github.com/minio/minio-go/v7"
gocache "github.com/patrickmn/go-cache"
)
// listObjectsBatch batch func for calling s3.ListObjects()
@ -20,9 +21,15 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
}
objCache, ok := c.Value("objCache").(*gocache.Cache)
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context"))
}
for _, v := range k {
results = append(results, &dataloader.Result{
Data: listObjects(s3Client, bucketName, v.String(), false),
Data: listObjects(s3Client, bucketName, v.String(), false, objCache),
})
}
@ -30,7 +37,7 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
}
// listObjects helper func for listObjectsBatch
func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool) []minio.ObjectInfo {
func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool, objCache *gocache.Cache) []minio.ObjectInfo {
objectCh := s3Client.ListObjects(context.Background(), bukitName, minio.ListObjectsOptions{
Prefix: path,
Recursive: false,
@ -39,6 +46,8 @@ func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool)
result := make([]minio.ObjectInfo, 0)
for obj := range objectCh {
// Add to object cache
objCache.SetDefault(obj.Key, obj)
result = append(result, obj)
}
@ -104,15 +113,18 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
}
for _, v := range k {
obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{})
objCache, ok := c.Value("objCache").(*gocache.Cache)
if err != nil {
results = append(results, &dataloader.Result{
Data: nil,
Error: err,
})
} else {
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context"))
}
for _, v := range k {
cachedObj, ok := objCache.Get(v.String())
if ok {
obj := cachedObj.(minio.ObjectInfo)
results = append(results, &dataloader.Result{
Data: &File{
ID: obj.Key,
@ -122,6 +134,25 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
},
Error: nil,
})
} else {
obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{})
if err != nil {
results = append(results, &dataloader.Result{
Data: nil,
Error: err,
})
} else {
results = append(results, &dataloader.Result{
Data: &File{
ID: obj.Key,
Size: obj.Size,
ContentType: obj.ContentType,
ETag: obj.ETag,
},
Error: nil,
})
}
}
}
@ -187,8 +218,8 @@ func handleLoaderError(k dataloader.Keys, err error) []*dataloader.Result {
return results
}
// createDataloader create all dataloaders and return a map of them
func createDataloader(config AppConfig) map[string]*dataloader.Loader {
// createDataloader create all dataloaders and return a map of them plus a cache for objects
func createDataloader(config AppConfig) (map[string]*dataloader.Loader, *gocache.Cache) {
loaderMap := make(map[string]*dataloader.Loader, 0)
loaderMap["getFiles"] = dataloader.NewBatchedLoader(getFilesBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
@ -196,5 +227,5 @@ func createDataloader(config AppConfig) map[string]*dataloader.Loader {
loaderMap["listObjects"] = dataloader.NewBatchedLoader(listObjectsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
loaderMap["getDirs"] = dataloader.NewBatchedLoader(getDirsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
return loaderMap
return loaderMap, gocache.New(config.CacheTTL, config.CacheCleanup)
}

87
internal/httpServer.go Normal file
View File

@ -0,0 +1,87 @@
package s3browser
import (
"context"
"fmt"
"io"
"mime"
"net/http"
"path/filepath"
"github.com/graphql-go/graphql"
"github.com/graphql-go/handler"
"github.com/minio/minio-go/v7"
)
// initHttp setup and start the http server. Blocking
func initHttp(schema graphql.Schema, resolveContext context.Context) {
h := handler.New(&handler.Config{
Schema: &schema,
Pretty: true,
GraphiQL: false,
Playground: true,
})
s3Client := resolveContext.Value("s3Client").(*minio.Client)
http.HandleFunc("/graphql", func(rw http.ResponseWriter, r *http.Request) {
h.ContextHandler(resolveContext, rw, r)
})
http.HandleFunc("/api/file", func(rw http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
httpGetFile(rw, r, s3Client)
return
}
if r.Method == "POST" {
httpPostFile(rw, r, s3Client)
return
}
})
http.ListenAndServe(":8080", nil)
}
func httpGetFile(rw http.ResponseWriter, r *http.Request, s3Client *minio.Client) {
id := r.URL.Query().Get("id")
objInfo, err := s3Client.StatObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
if err != nil {
rw.WriteHeader(500)
return
}
reqEtag := r.Header.Get("If-None-Match")
if reqEtag == objInfo.ETag {
rw.WriteHeader(304)
return
}
obj, err := s3Client.GetObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
if err != nil {
rw.WriteHeader(500)
return
}
rw.Header().Set("Cache-Control", "must-revalidate")
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key))))
rw.Header().Set("Content-Type", objInfo.ContentType)
rw.Header().Set("ETag", objInfo.ETag)
io.Copy(rw, obj)
}
func httpPostFile(rw http.ResponseWriter, r *http.Request, s3Client *minio.Client) {
contentType := r.Header.Get("Content-Type")
mimeType, err, _ := mime.ParseMediaType(contentType)
if mimeType != "multipart/form-data" || err != nil {
rw.WriteHeader(400)
return
}
r.ParseMultipartForm(10 << 20)
}

64
internal/s3Broswer.go
View File

@ -2,16 +2,9 @@ package s3browser
import (
"context"
"fmt"
"io"
"log"
"net/http"
"path/filepath"
"time"
"github.com/graph-gophers/dataloader"
"github.com/graphql-go/graphql"
"github.com/graphql-go/handler"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
@ -46,55 +39,6 @@ type Directory struct {
var bucketName = "dev"
// initHttp setup and start the http server. Blocking
func initHttp(schema graphql.Schema, s3Client *minio.Client, loaderMap map[string]*dataloader.Loader) {
h := handler.New(&handler.Config{
Schema: &schema,
Pretty: true,
GraphiQL: false,
Playground: true,
})
resolveContext := context.WithValue(context.Background(), "s3Client", s3Client)
resolveContext = context.WithValue(resolveContext, "loader", loaderMap)
http.HandleFunc("/graphql", func(rw http.ResponseWriter, r *http.Request) {
h.ContextHandler(resolveContext, rw, r)
})
http.HandleFunc("/api/file", func(rw http.ResponseWriter, r *http.Request) {
id := r.URL.Query().Get("id")
objInfo, err := s3Client.StatObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
if err != nil {
rw.WriteHeader(500)
return
}
reqEtag := r.Header.Get("If-None-Match")
if reqEtag == objInfo.ETag {
rw.WriteHeader(304)
return
}
obj, err := s3Client.GetObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
if err != nil {
rw.WriteHeader(500)
return
}
rw.Header().Set("Cache-Control", "must-revalidate")
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key))))
rw.Header().Set("Content-Type", objInfo.ContentType)
rw.Header().Set("ETag", objInfo.ETag)
io.Copy(rw, obj)
})
http.ListenAndServe(":8080", nil)
}
// setupS3Client connect the s3Client
func setupS3Client(config AppConfig) *minio.Client {
minioClient, err := minio.New(config.S3Endoint, &minio.Options{
@ -125,7 +69,7 @@ func setupS3Client(config AppConfig) *minio.Client {
func Start(config AppConfig) {
s3Client := setupS3Client(config)
loader := createDataloader(config)
loaderMap, objCache := createDataloader(config)
graphqlTypes()
schema, err := graphqlSchema()
@ -134,5 +78,9 @@ func Start(config AppConfig) {
log.Panic(err)
}
initHttp(schema, s3Client, loader)
resolveContext := context.WithValue(context.Background(), "s3Client", s3Client)
resolveContext = context.WithValue(resolveContext, "loader", loaderMap)
resolveContext = context.WithValue(resolveContext, "objCache", objCache)
initHttp(schema, resolveContext)
}