package loader import ( "context" "fmt" "path/filepath" "strings" "git.kapelle.org/niklas/s3browser/internal/cache" "git.kapelle.org/niklas/s3browser/internal/helper" types "git.kapelle.org/niklas/s3browser/internal/types" "github.com/graph-gophers/dataloader" "github.com/minio/minio-go/v7" ) type Loader struct { listObjectsLoader *dataloader.Loader listObjectsRecursiveLoader *dataloader.Loader statObjectLoader *dataloader.Loader listBucketsLoader *dataloader.Loader listObjectsLoaderCache cache.S3Cache listObjectsRecursiveLoaderCache cache.S3Cache statObjectLoaderCache cache.S3Cache listBucketsLoaderCache cache.S3Cache } func NewLoader(config types.AppConfig) *Loader { listObjectsLoaderCache := &dataloader.NoCache{} listObjectsRecursiveLoaderCache := &dataloader.NoCache{} statObjectLoaderCache := cache.NewTTLCache(config.CacheTTL, config.CacheCleanup) listBucketsLoaderCache := cache.NewTTLCache(config.CacheTTL, config.CacheCleanup) return &Loader{ listObjectsLoader: dataloader.NewBatchedLoader( listObjectsBatch, dataloader.WithCache(listObjectsLoaderCache), ), listObjectsLoaderCache: listObjectsLoaderCache, listObjectsRecursiveLoader: dataloader.NewBatchedLoader( listObjectsRecursiveBatch, dataloader.WithCache(listObjectsRecursiveLoaderCache), ), listObjectsRecursiveLoaderCache: listObjectsRecursiveLoaderCache, statObjectLoader: dataloader.NewBatchedLoader( statObjectBatch, dataloader.WithCache(statObjectLoaderCache), ), statObjectLoaderCache: statObjectLoaderCache, listBucketsLoader: dataloader.NewBatchedLoader( listBucketsBatch, dataloader.WithCache(listBucketsLoaderCache), ), listBucketsLoaderCache: listBucketsLoaderCache, } } func (l *Loader) GetFiles(ctx context.Context, path types.ID) ([]types.File, error) { thunk := l.listObjectsLoader.Load(ctx, path) objects, err := thunk() if err != nil { return nil, err } var files []types.File for _, obj := range objects.([]minio.ObjectInfo) { if obj.Err != nil { return nil, obj.Err } else if !strings.HasSuffix(obj.Key, "/") { files = append(files, *helper.ObjInfoToFile(obj, path.Bucket)) } } return files, nil } func (l *Loader) GetFile(ctx context.Context, id types.ID) (*types.File, error) { thunk := l.statObjectLoader.Load(ctx, id) result, err := thunk() if err != nil { return nil, err } objInfo, ok := result.(minio.ObjectInfo) if !ok { return nil, fmt.Errorf("Failed to stats object") } return helper.ObjInfoToFile(objInfo, id.Bucket), nil } func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory, error) { thunk := l.listObjectsLoader.Load(ctx, path) result, err := thunk() if err != nil { return nil, err } var dirs []types.Directory for _, obj := range result.([]minio.ObjectInfo) { if obj.Err != nil { return nil, obj.Err } else if strings.HasSuffix(obj.Key, "/") { resultID := types.ID{ Bucket: path.Bucket, Key: obj.Key, } resultID.Normalize() dirs = append(dirs, types.Directory{ ID: resultID, Name: filepath.Base(obj.Key), }) } } return dirs, nil } func (l *Loader) GetBuckets(ctx context.Context) ([]string, error) { thunk := l.listBucketsLoader.Load(ctx, dataloader.StringKey("")) result, err := thunk() if err != nil { return nil, err } bucketsInfo := result.([]minio.BucketInfo) var buckets []string for _, i := range bucketsInfo { buckets = append(buckets, i.Name) } return buckets, nil } func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.File, error) { thunk := l.listObjectsRecursiveLoader.Load(ctx, path) result, err := thunk() if err != nil { return nil, err } objects := result.([]minio.ObjectInfo) var files []types.File for _, obj := range objects { files = append(files, *helper.ObjInfoToFile(obj, path.Bucket)) } return files, nil } func (l *Loader) InvalidedCacheForId(ctx context.Context, id types.ID) { parent := id.Parent() l.statObjectLoader.Clear(ctx, id) // Code below is useless for now until we use a propper cache for "listObjectsLoader" and "listObjectsRecursiveLoader" // TODO: implement cache invalidation for "listObjectsLoader" and "listObjectsRecursiveLoader" l.listObjectsLoader.Clear(ctx, id).Clear(ctx, parent) // Remove up from recursive list for rParent := parent; rParent != nil; rParent = rParent.Parent() { l.listObjectsRecursiveLoader.Clear(ctx, rParent) } }