From 6598088024e4ec87f528f34a4b44e7e11bd94a29 Mon Sep 17 00:00:00 2001 From: Djeeberjr Date: Tue, 3 Aug 2021 23:10:23 +0200 Subject: [PATCH] MORE CACHES --- internal/dataloader.go | 57 ++++++++++++++++++++------- internal/httpServer.go | 87 ++++++++++++++++++++++++++++++++++++++++++ internal/s3Broswer.go | 64 +++---------------------------- 3 files changed, 137 insertions(+), 71 deletions(-) create mode 100644 internal/httpServer.go diff --git a/internal/dataloader.go b/internal/dataloader.go index e1c600b..2d131eb 100644 --- a/internal/dataloader.go +++ b/internal/dataloader.go @@ -8,6 +8,7 @@ import ( "github.com/graph-gophers/dataloader" "github.com/minio/minio-go/v7" + gocache "github.com/patrickmn/go-cache" ) // listObjectsBatch batch func for calling s3.ListObjects() @@ -20,9 +21,15 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) } + objCache, ok := c.Value("objCache").(*gocache.Cache) + + if !ok { + return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context")) + } + for _, v := range k { results = append(results, &dataloader.Result{ - Data: listObjects(s3Client, bucketName, v.String(), false), + Data: listObjects(s3Client, bucketName, v.String(), false, objCache), }) } @@ -30,7 +37,7 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result } // listObjects helper func for listObjectsBatch -func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool) []minio.ObjectInfo { +func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool, objCache *gocache.Cache) []minio.ObjectInfo { objectCh := s3Client.ListObjects(context.Background(), bukitName, minio.ListObjectsOptions{ Prefix: path, Recursive: false, @@ -39,6 +46,8 @@ func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool) result := make([]minio.ObjectInfo, 0) for obj := range objectCh { + // Add to object cache + objCache.SetDefault(obj.Key, obj) result = append(result, obj) } @@ -104,15 +113,18 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result { return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) } - for _, v := range k { - obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{}) + objCache, ok := c.Value("objCache").(*gocache.Cache) - if err != nil { - results = append(results, &dataloader.Result{ - Data: nil, - Error: err, - }) - } else { + if !ok { + return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context")) + } + + for _, v := range k { + + cachedObj, ok := objCache.Get(v.String()) + + if ok { + obj := cachedObj.(minio.ObjectInfo) results = append(results, &dataloader.Result{ Data: &File{ ID: obj.Key, @@ -122,6 +134,25 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result { }, Error: nil, }) + } else { + obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{}) + + if err != nil { + results = append(results, &dataloader.Result{ + Data: nil, + Error: err, + }) + } else { + results = append(results, &dataloader.Result{ + Data: &File{ + ID: obj.Key, + Size: obj.Size, + ContentType: obj.ContentType, + ETag: obj.ETag, + }, + Error: nil, + }) + } } } @@ -187,8 +218,8 @@ func handleLoaderError(k dataloader.Keys, err error) []*dataloader.Result { return results } -// createDataloader create all dataloaders and return a map of them -func createDataloader(config AppConfig) map[string]*dataloader.Loader { +// createDataloader create all dataloaders and return a map of them plus a cache for objects +func createDataloader(config AppConfig) (map[string]*dataloader.Loader, *gocache.Cache) { loaderMap := make(map[string]*dataloader.Loader, 0) loaderMap["getFiles"] = dataloader.NewBatchedLoader(getFilesBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup))) @@ -196,5 +227,5 @@ func createDataloader(config AppConfig) map[string]*dataloader.Loader { loaderMap["listObjects"] = dataloader.NewBatchedLoader(listObjectsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup))) loaderMap["getDirs"] = dataloader.NewBatchedLoader(getDirsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup))) - return loaderMap + return loaderMap, gocache.New(config.CacheTTL, config.CacheCleanup) } diff --git a/internal/httpServer.go b/internal/httpServer.go new file mode 100644 index 0000000..8643e13 --- /dev/null +++ b/internal/httpServer.go @@ -0,0 +1,87 @@ +package s3browser + +import ( + "context" + "fmt" + "io" + "mime" + "net/http" + "path/filepath" + + "github.com/graphql-go/graphql" + "github.com/graphql-go/handler" + "github.com/minio/minio-go/v7" +) + +// initHttp setup and start the http server. Blocking +func initHttp(schema graphql.Schema, resolveContext context.Context) { + h := handler.New(&handler.Config{ + Schema: &schema, + Pretty: true, + GraphiQL: false, + Playground: true, + }) + + s3Client := resolveContext.Value("s3Client").(*minio.Client) + + http.HandleFunc("/graphql", func(rw http.ResponseWriter, r *http.Request) { + h.ContextHandler(resolveContext, rw, r) + }) + + http.HandleFunc("/api/file", func(rw http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + httpGetFile(rw, r, s3Client) + return + } + + if r.Method == "POST" { + httpPostFile(rw, r, s3Client) + return + } + }) + + http.ListenAndServe(":8080", nil) +} + +func httpGetFile(rw http.ResponseWriter, r *http.Request, s3Client *minio.Client) { + id := r.URL.Query().Get("id") + objInfo, err := s3Client.StatObject(context.Background(), bucketName, id, minio.GetObjectOptions{}) + + if err != nil { + rw.WriteHeader(500) + return + } + + reqEtag := r.Header.Get("If-None-Match") + if reqEtag == objInfo.ETag { + rw.WriteHeader(304) + return + } + + obj, err := s3Client.GetObject(context.Background(), bucketName, id, minio.GetObjectOptions{}) + + if err != nil { + rw.WriteHeader(500) + return + } + + rw.Header().Set("Cache-Control", "must-revalidate") + rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key)))) + rw.Header().Set("Content-Type", objInfo.ContentType) + rw.Header().Set("ETag", objInfo.ETag) + + io.Copy(rw, obj) +} + +func httpPostFile(rw http.ResponseWriter, r *http.Request, s3Client *minio.Client) { + + contentType := r.Header.Get("Content-Type") + mimeType, err, _ := mime.ParseMediaType(contentType) + if mimeType != "multipart/form-data" || err != nil { + rw.WriteHeader(400) + return + } + + r.ParseMultipartForm(10 << 20) + +} diff --git a/internal/s3Broswer.go b/internal/s3Broswer.go index 56a81fe..b08cb39 100644 --- a/internal/s3Broswer.go +++ b/internal/s3Broswer.go @@ -2,16 +2,9 @@ package s3browser import ( "context" - "fmt" - "io" "log" - "net/http" - "path/filepath" "time" - "github.com/graph-gophers/dataloader" - "github.com/graphql-go/graphql" - "github.com/graphql-go/handler" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) @@ -46,55 +39,6 @@ type Directory struct { var bucketName = "dev" -// initHttp setup and start the http server. Blocking -func initHttp(schema graphql.Schema, s3Client *minio.Client, loaderMap map[string]*dataloader.Loader) { - h := handler.New(&handler.Config{ - Schema: &schema, - Pretty: true, - GraphiQL: false, - Playground: true, - }) - - resolveContext := context.WithValue(context.Background(), "s3Client", s3Client) - resolveContext = context.WithValue(resolveContext, "loader", loaderMap) - - http.HandleFunc("/graphql", func(rw http.ResponseWriter, r *http.Request) { - h.ContextHandler(resolveContext, rw, r) - }) - - http.HandleFunc("/api/file", func(rw http.ResponseWriter, r *http.Request) { - id := r.URL.Query().Get("id") - objInfo, err := s3Client.StatObject(context.Background(), bucketName, id, minio.GetObjectOptions{}) - - if err != nil { - rw.WriteHeader(500) - return - } - - reqEtag := r.Header.Get("If-None-Match") - if reqEtag == objInfo.ETag { - rw.WriteHeader(304) - return - } - - obj, err := s3Client.GetObject(context.Background(), bucketName, id, minio.GetObjectOptions{}) - - if err != nil { - rw.WriteHeader(500) - return - } - - rw.Header().Set("Cache-Control", "must-revalidate") - rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key)))) - rw.Header().Set("Content-Type", objInfo.ContentType) - rw.Header().Set("ETag", objInfo.ETag) - - io.Copy(rw, obj) - }) - - http.ListenAndServe(":8080", nil) -} - // setupS3Client connect the s3Client func setupS3Client(config AppConfig) *minio.Client { minioClient, err := minio.New(config.S3Endoint, &minio.Options{ @@ -125,7 +69,7 @@ func setupS3Client(config AppConfig) *minio.Client { func Start(config AppConfig) { s3Client := setupS3Client(config) - loader := createDataloader(config) + loaderMap, objCache := createDataloader(config) graphqlTypes() schema, err := graphqlSchema() @@ -134,5 +78,9 @@ func Start(config AppConfig) { log.Panic(err) } - initHttp(schema, s3Client, loader) + resolveContext := context.WithValue(context.Background(), "s3Client", s3Client) + resolveContext = context.WithValue(resolveContext, "loader", loaderMap) + resolveContext = context.WithValue(resolveContext, "objCache", objCache) + + initHttp(schema, resolveContext) }