MORE CACHES

internal/dataloader.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/graph-gophers/dataloader"
 	"github.com/minio/minio-go/v7"
+	gocache "github.com/patrickmn/go-cache"
 )
 
 // listObjectsBatch batch func for calling s3.ListObjects()
@@ -20,9 +21,15 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
 		return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
 	}
 
+	objCache, ok := c.Value("objCache").(*gocache.Cache)
+
+	if !ok {
+		return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context"))
+	}
+
 	for _, v := range k {
 		results = append(results, &dataloader.Result{
-			Data: listObjects(s3Client, bucketName, v.String(), false),
+			Data: listObjects(s3Client, bucketName, v.String(), false, objCache),
 		})
 	}
 
@@ -30,7 +37,7 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
 }
 
 // listObjects helper func for listObjectsBatch
-func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool) []minio.ObjectInfo {
+func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool, objCache *gocache.Cache) []minio.ObjectInfo {
 	objectCh := s3Client.ListObjects(context.Background(), bukitName, minio.ListObjectsOptions{
 		Prefix:    path,
 		Recursive: false,
@@ -39,6 +46,8 @@ func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool)
 	result := make([]minio.ObjectInfo, 0)
 
 	for obj := range objectCh {
+		// Add to object cache
+		objCache.SetDefault(obj.Key, obj)
 		result = append(result, obj)
 	}
 
@@ -104,15 +113,18 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
 		return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
 	}
 
-	for _, v := range k {
+	objCache, ok := c.Value("objCache").(*gocache.Cache)
-		obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{})
 
-		if err != nil {
+	if !ok {
-			results = append(results, &dataloader.Result{
+		return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context"))
-				Data:  nil,
+	}
-				Error: err,
+
-			})
+	for _, v := range k {
-		} else {
+
+		cachedObj, ok := objCache.Get(v.String())
+
+		if ok {
+			obj := cachedObj.(minio.ObjectInfo)
 			results = append(results, &dataloader.Result{
 				Data: &File{
 					ID:          obj.Key,
@@ -122,6 +134,25 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
 				},
 				Error: nil,
 			})
+		} else {
+			obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{})
+
+			if err != nil {
+				results = append(results, &dataloader.Result{
+					Data:  nil,
+					Error: err,
+				})
+			} else {
+				results = append(results, &dataloader.Result{
+					Data: &File{
+						ID:          obj.Key,
+						Size:        obj.Size,
+						ContentType: obj.ContentType,
+						ETag:        obj.ETag,
+					},
+					Error: nil,
+				})
+			}
 		}
 	}
 
@@ -187,8 +218,8 @@ func handleLoaderError(k dataloader.Keys, err error) []*dataloader.Result {
 	return results
 }
 
-// createDataloader create all dataloaders and return a map of them
+// createDataloader create all dataloaders and return a map of them plus a cache for objects
-func createDataloader(config AppConfig) map[string]*dataloader.Loader {
+func createDataloader(config AppConfig) (map[string]*dataloader.Loader, *gocache.Cache) {
 	loaderMap := make(map[string]*dataloader.Loader, 0)
 
 	loaderMap["getFiles"] = dataloader.NewBatchedLoader(getFilesBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
@@ -196,5 +227,5 @@ func createDataloader(config AppConfig) map[string]*dataloader.Loader {
 	loaderMap["listObjects"] = dataloader.NewBatchedLoader(listObjectsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
 	loaderMap["getDirs"] = dataloader.NewBatchedLoader(getDirsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
 
-	return loaderMap
+	return loaderMap, gocache.New(config.CacheTTL, config.CacheCleanup)
 }

internal/httpServer.go

@@ -0,0 +1,87 @@
+package s3browser
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"path/filepath"
+
+	"github.com/graphql-go/graphql"
+	"github.com/graphql-go/handler"
+	"github.com/minio/minio-go/v7"
+)
+
+// initHttp setup and start the http server. Blocking
+func initHttp(schema graphql.Schema, resolveContext context.Context) {
+	h := handler.New(&handler.Config{
+		Schema:     &schema,
+		Pretty:     true,
+		GraphiQL:   false,
+		Playground: true,
+	})
+
+	s3Client := resolveContext.Value("s3Client").(*minio.Client)
+
+	http.HandleFunc("/graphql", func(rw http.ResponseWriter, r *http.Request) {
+		h.ContextHandler(resolveContext, rw, r)
+	})
+
+	http.HandleFunc("/api/file", func(rw http.ResponseWriter, r *http.Request) {
+		if r.Method == "GET" {
+			httpGetFile(rw, r, s3Client)
+			return
+		}
+
+		if r.Method == "POST" {
+			httpPostFile(rw, r, s3Client)
+			return
+		}
+	})
+
+	http.ListenAndServe(":8080", nil)
+}
+
+func httpGetFile(rw http.ResponseWriter, r *http.Request, s3Client *minio.Client) {
+	id := r.URL.Query().Get("id")
+	objInfo, err := s3Client.StatObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
+
+	if err != nil {
+		rw.WriteHeader(500)
+		return
+	}
+
+	reqEtag := r.Header.Get("If-None-Match")
+	if reqEtag == objInfo.ETag {
+		rw.WriteHeader(304)
+		return
+	}
+
+	obj, err := s3Client.GetObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
+
+	if err != nil {
+		rw.WriteHeader(500)
+		return
+	}
+
+	rw.Header().Set("Cache-Control", "must-revalidate")
+	rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key))))
+	rw.Header().Set("Content-Type", objInfo.ContentType)
+	rw.Header().Set("ETag", objInfo.ETag)
+
+	io.Copy(rw, obj)
+}
+
+func httpPostFile(rw http.ResponseWriter, r *http.Request, s3Client *minio.Client) {
+
+	contentType := r.Header.Get("Content-Type")
+	mimeType, err, _ := mime.ParseMediaType(contentType)
+	if mimeType != "multipart/form-data" || err != nil {
+		rw.WriteHeader(400)
+		return
+	}
+
+	r.ParseMultipartForm(10 << 20)
+
+}

internal/s3Broswer.go

@@ -2,16 +2,9 @@ package s3browser
 
 import (
 	"context"
-	"fmt"
-	"io"
 	"log"
-	"net/http"
-	"path/filepath"
 	"time"
 
-	"github.com/graph-gophers/dataloader"
-	"github.com/graphql-go/graphql"
-	"github.com/graphql-go/handler"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
@@ -46,55 +39,6 @@ type Directory struct {
 
 var bucketName = "dev"
 
-// initHttp setup and start the http server. Blocking
-func initHttp(schema graphql.Schema, s3Client *minio.Client, loaderMap map[string]*dataloader.Loader) {
-	h := handler.New(&handler.Config{
-		Schema:     &schema,
-		Pretty:     true,
-		GraphiQL:   false,
-		Playground: true,
-	})
-
-	resolveContext := context.WithValue(context.Background(), "s3Client", s3Client)
-	resolveContext = context.WithValue(resolveContext, "loader", loaderMap)
-
-	http.HandleFunc("/graphql", func(rw http.ResponseWriter, r *http.Request) {
-		h.ContextHandler(resolveContext, rw, r)
-	})
-
-	http.HandleFunc("/api/file", func(rw http.ResponseWriter, r *http.Request) {
-		id := r.URL.Query().Get("id")
-		objInfo, err := s3Client.StatObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
-
-		if err != nil {
-			rw.WriteHeader(500)
-			return
-		}
-
-		reqEtag := r.Header.Get("If-None-Match")
-		if reqEtag == objInfo.ETag {
-			rw.WriteHeader(304)
-			return
-		}
-
-		obj, err := s3Client.GetObject(context.Background(), bucketName, id, minio.GetObjectOptions{})
-
-		if err != nil {
-			rw.WriteHeader(500)
-			return
-		}
-
-		rw.Header().Set("Cache-Control", "must-revalidate")
-		rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key))))
-		rw.Header().Set("Content-Type", objInfo.ContentType)
-		rw.Header().Set("ETag", objInfo.ETag)
-
-		io.Copy(rw, obj)
-	})
-
-	http.ListenAndServe(":8080", nil)
-}
-
 // setupS3Client connect the s3Client
 func setupS3Client(config AppConfig) *minio.Client {
 	minioClient, err := minio.New(config.S3Endoint, &minio.Options{
@@ -125,7 +69,7 @@ func setupS3Client(config AppConfig) *minio.Client {
 func Start(config AppConfig) {
 	s3Client := setupS3Client(config)
 
-	loader := createDataloader(config)
+	loaderMap, objCache := createDataloader(config)
 
 	graphqlTypes()
 	schema, err := graphqlSchema()
@@ -134,5 +78,9 @@ func Start(config AppConfig) {
 		log.Panic(err)
 	}
 
-	initHttp(schema, s3Client, loader)
+	resolveContext := context.WithValue(context.Background(), "s3Client", s3Client)
+	resolveContext = context.WithValue(resolveContext, "loader", loaderMap)
+	resolveContext = context.WithValue(resolveContext, "objCache", objCache)
+
+	initHttp(schema, resolveContext)
 }