niklas/s3browser-backend
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

LESS CACHES

Browse Source
This commit is contained in:
Djeeberjr 2021-08-06 13:48:49 +02:00
parent 6598088024
commit b42f45afe9
2 changed files with 13 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
internal/dataloader.go
View File

@ -8,7 +8,6 @@ import (
"github.com/graph-gophers/dataloader" "github.com/graph-gophers/dataloader"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
gocache "github.com/patrickmn/go-cache"
) )
// listObjectsBatch batch func for calling s3.ListObjects() // listObjectsBatch batch func for calling s3.ListObjects()
@ -21,15 +20,9 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
} }
objCache, ok := c.Value("objCache").(*gocache.Cache)
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context"))
}
for _, v := range k { for _, v := range k {
results = append(results, &dataloader.Result{ results = append(results, &dataloader.Result{
Data: listObjects(s3Client, bucketName, v.String(), false, objCache), Data: listObjects(s3Client, bucketName, v.String(), false),
}) })
} }
@ -37,17 +30,14 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
} }
// listObjects helper func for listObjectsBatch // listObjects helper func for listObjectsBatch
func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool, objCache *gocache.Cache) []minio.ObjectInfo { func listObjects(s3Client *minio.Client, bukitName, path string, recursive bool) []minio.ObjectInfo {
objectCh := s3Client.ListObjects(context.Background(), bukitName, minio.ListObjectsOptions{ objectCh := s3Client.ListObjects(context.Background(), bukitName, minio.ListObjectsOptions{
Prefix: path, Prefix: path,
Recursive: false, Recursive: false,
}) })
result := make([]minio.ObjectInfo, 0) result := make([]minio.ObjectInfo, 0)
for obj := range objectCh { for obj := range objectCh {
// Add to object cache
objCache.SetDefault(obj.Key, obj)
result = append(result, obj) result = append(result, obj)
} }
@ -113,28 +103,8 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
} }
objCache, ok := c.Value("objCache").(*gocache.Cache)
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get objCache from context"))
}
for _, v := range k { for _, v := range k {
cachedObj, ok := objCache.Get(v.String())
if ok {
obj := cachedObj.(minio.ObjectInfo)
results = append(results, &dataloader.Result{
Data: &File{
ID: obj.Key,
Size: obj.Size,
ContentType: obj.ContentType,
ETag: obj.ETag,
},
Error: nil,
})
} else {
obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{}) obj, err := s3Client.StatObject(context.Background(), bucketName, v.String(), minio.StatObjectOptions{})
if err != nil { if err != nil {
@ -153,7 +123,7 @@ func getFileBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
Error: nil, Error: nil,
}) })
} }
}
} }
return results return results
@ -219,7 +189,7 @@ func handleLoaderError(k dataloader.Keys, err error) []*dataloader.Result {
} }
// createDataloader create all dataloaders and return a map of them plus a cache for objects // createDataloader create all dataloaders and return a map of them plus a cache for objects
func createDataloader(config AppConfig) (map[string]*dataloader.Loader, *gocache.Cache) { func createDataloader(config AppConfig) map[string]*dataloader.Loader {
loaderMap := make(map[string]*dataloader.Loader, 0) loaderMap := make(map[string]*dataloader.Loader, 0)
loaderMap["getFiles"] = dataloader.NewBatchedLoader(getFilesBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup))) loaderMap["getFiles"] = dataloader.NewBatchedLoader(getFilesBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
@ -227,5 +197,5 @@ func createDataloader(config AppConfig) (map[string]*dataloader.Loader, *gocache
loaderMap["listObjects"] = dataloader.NewBatchedLoader(listObjectsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup))) loaderMap["listObjects"] = dataloader.NewBatchedLoader(listObjectsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
loaderMap["getDirs"] = dataloader.NewBatchedLoader(getDirsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup))) loaderMap["getDirs"] = dataloader.NewBatchedLoader(getDirsBatch, dataloader.WithCache(newCache(config.CacheTTL, config.CacheCleanup)))
return loaderMap, gocache.New(config.CacheTTL, config.CacheCleanup) return loaderMap
} }

3
internal/s3Broswer.go
View File

@ -69,7 +69,7 @@ func setupS3Client(config AppConfig) *minio.Client {
func Start(config AppConfig) { func Start(config AppConfig) {
s3Client := setupS3Client(config) s3Client := setupS3Client(config)
loaderMap, objCache := createDataloader(config) loaderMap := createDataloader(config)
graphqlTypes() graphqlTypes()
schema, err := graphqlSchema() schema, err := graphqlSchema()
@ -80,7 +80,6 @@ func Start(config AppConfig) {
resolveContext := context.WithValue(context.Background(), "s3Client", s3Client) resolveContext := context.WithValue(context.Background(), "s3Client", s3Client)
resolveContext = context.WithValue(resolveContext, "loader", loaderMap) resolveContext = context.WithValue(resolveContext, "loader", loaderMap)
resolveContext = context.WithValue(resolveContext, "objCache", objCache)
initHttp(schema, resolveContext) initHttp(schema, resolveContext)
} }