niklas/s3browser-backend
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use s3Service interface

Browse Source
This commit is contained in:
Djeeberjr
2021-11-23 20:12:24 +01:00
parent 60817c2249
commit 8d85d645d6
9 changed files with 162 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
internal/loader/batch.go
View File

@@ -4,9 +4,9 @@ import (
"context"
"fmt"
types "git.kapelle.org/niklas/s3browser/internal/types"
"git.kapelle.org/niklas/s3browser/internal/s3"
"git.kapelle.org/niklas/s3browser/internal/types"
"github.com/graph-gophers/dataloader"
"github.com/minio/minio-go/v7"
log "github.com/sirupsen/logrus"
)
@@ -14,7 +14,7 @@ import (
func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
var results []*dataloader.Result
s3Client, ok := c.Value("s3Client").(*minio.Client)
s3Client, ok := c.Value("s3Client").(s3.S3Service)
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@@ -22,10 +22,18 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
for _, v := range k {
id := v.Raw().(types.ID)
results = append(results, &dataloader.Result{
Data: listObjects(s3Client, id, false),
Error: nil,
})
objects, err := s3Client.ListObjects(c, id)
if err != nil {
results = append(results, &dataloader.Result{
Data: nil,
Error: err,
})
} else {
results = append(results, &dataloader.Result{
Data: objects,
Error: nil,
})
}
}
return results
@@ -35,7 +43,7 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
var results []*dataloader.Result
s3Client, ok := c.Value("s3Client").(*minio.Client)
s3Client, ok := c.Value("s3Client").(s3.S3Service)
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@@ -43,35 +51,27 @@ func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataload
for _, v := range k {
id := v.Raw().(types.ID)
results = append(results, &dataloader.Result{
Data: listObjects(s3Client, id, true),
Error: nil,
})
objects, err := s3Client.ListObjectsRecursive(c, id)
if err != nil {
results = append(results, &dataloader.Result{
Data: nil,
Error: err,
})
} else {
results = append(results, &dataloader.Result{
Data: objects,
Error: nil,
})
}
}
return results
}
// listObjects helper func for listObjectsBatch
func listObjects(s3Client *minio.Client, id types.ID, recursive bool) []minio.ObjectInfo {
log.Debug("S3 'ListObjects': ", id)
objectCh := s3Client.ListObjects(context.Background(), id.Bucket, minio.ListObjectsOptions{
Prefix: id.Key,
Recursive: recursive,
})
result := make([]minio.ObjectInfo, 0)
for obj := range objectCh {
result = append(result, obj)
}
return result
}
func listBucketsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
var results []*dataloader.Result
s3Client, ok := c.Value("s3Client").(*minio.Client)
s3Client, ok := c.Value("s3Client").(s3.S3Service)
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@@ -100,7 +100,7 @@ func statObjectBatch(ctx context.Context, k dataloader.Keys) []*dataloader.Resul
log.Debug("statObjectBatch")
var results []*dataloader.Result
s3Client, ok := ctx.Value("s3Client").(*minio.Client)
s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@@ -109,7 +109,7 @@ func statObjectBatch(ctx context.Context, k dataloader.Keys) []*dataloader.Resul
for _, v := range k {
id := v.Raw().(types.ID)
log.Debug("S3 'StatObject': ", id)
stat, err := s3Client.StatObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{})
stat, err := s3Client.StatObject(ctx, id)
results = append(results, &dataloader.Result{
Data: stat,
Error: err,

40
internal/loader/loader.go
View File

@@ -3,14 +3,12 @@ package loader
import (
"context"
"fmt"
"path/filepath"
"strings"
"git.kapelle.org/niklas/s3browser/internal/cache"
"git.kapelle.org/niklas/s3browser/internal/helper"
"git.kapelle.org/niklas/s3browser/internal/s3"
types "git.kapelle.org/niklas/s3browser/internal/types"
"github.com/graph-gophers/dataloader"
"github.com/minio/minio-go/v7"
)
type Loader struct {
@@ -68,10 +66,8 @@ func (l *Loader) GetFiles(ctx context.Context, path types.ID) ([]types.File, err
var files []types.File
for _, obj := range objects.([]minio.ObjectInfo) {
if obj.Err != nil {
return nil, obj.Err
} else if !strings.HasSuffix(obj.Key, "/") {
for _, obj := range objects.([]s3.Object) {
if !obj.ID.IsDirectory() {
files = append(files, *helper.ObjInfoToFile(obj, path.Bucket))
}
}
@@ -86,13 +82,13 @@ func (l *Loader) GetFile(ctx context.Context, id types.ID) (*types.File, error)
if err != nil {
return nil, err
}
objInfo, ok := result.(minio.ObjectInfo)
objInfo, ok := result.(*s3.Object)
if !ok {
return nil, fmt.Errorf("Failed to stats object")
}
return helper.ObjInfoToFile(objInfo, id.Bucket), nil
return helper.ObjInfoToFile(*objInfo, id.Bucket), nil
}
func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory, error) {
@@ -104,20 +100,10 @@ func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory,
}
var dirs []types.Directory
for _, obj := range result.([]minio.ObjectInfo) {
if obj.Err != nil {
return nil, obj.Err
} else if strings.HasSuffix(obj.Key, "/") {
resultID := types.ID{
Bucket: path.Bucket,
Key: obj.Key,
}
resultID.Normalize()
for _, obj := range result.([]s3.Object) {
if obj.ID.IsDirectory() {
dirs = append(dirs, types.Directory{
ID: resultID,
Name: filepath.Base(obj.Key),
ID: obj.ID,
})
}
}
@@ -134,13 +120,7 @@ func (l *Loader) GetBuckets(ctx context.Context) ([]string, error) {
return nil, err
}
bucketsInfo := result.([]minio.BucketInfo)
var buckets []string
for _, i := range bucketsInfo {
buckets = append(buckets, i.Name)
}
return buckets, nil
return result.([]string), nil
}
func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.File, error) {
@@ -151,7 +131,7 @@ func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.
return nil, err
}
objects := result.([]minio.ObjectInfo)
objects := result.([]s3.Object)
var files []types.File
for _, obj := range objects {