niklas/s3browser-backend
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use s3Service interface

Browse Source
This commit is contained in:
Djeeberjr 2021-11-23 20:12:24 +01:00
parent 60817c2249
commit 8d85d645d6
9 changed files with 162 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
internal/gql/mutations.go
View File

@ -5,17 +5,16 @@ import (
"fmt" "fmt"
"strings" "strings"
"github.com/minio/minio-go/v7"
"git.kapelle.org/niklas/s3browser/internal/db" "git.kapelle.org/niklas/s3browser/internal/db"
helper "git.kapelle.org/niklas/s3browser/internal/helper" helper "git.kapelle.org/niklas/s3browser/internal/helper"
"git.kapelle.org/niklas/s3browser/internal/loader" "git.kapelle.org/niklas/s3browser/internal/loader"
"git.kapelle.org/niklas/s3browser/internal/s3"
types "git.kapelle.org/niklas/s3browser/internal/types" types "git.kapelle.org/niklas/s3browser/internal/types"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
func deleteMutation(ctx context.Context, id types.ID) error { func deleteMutation(ctx context.Context, id types.ID) error {
s3Client, ok := ctx.Value("s3Client").(*minio.Client) s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return fmt.Errorf("Failed to get s3Client from context") return fmt.Errorf("Failed to get s3Client from context")
@ -24,7 +23,7 @@ func deleteMutation(ctx context.Context, id types.ID) error {
log.Debug("S3 'RemoveObject': ", id) log.Debug("S3 'RemoveObject': ", id)
// TODO: it is posible to remove multiple objects with a single call. // TODO: it is posible to remove multiple objects with a single call.
// Is it better to batch this? // Is it better to batch this?
err := s3Client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{}) err := s3Client.RemoveObject(ctx, id)
if err != nil { if err != nil {
return err return err
@ -36,7 +35,7 @@ func deleteMutation(ctx context.Context, id types.ID) error {
} }
func copyMutation(ctx context.Context, src, dest types.ID) (*types.File, error) { func copyMutation(ctx context.Context, src, dest types.ID) (*types.File, error) {
s3Client, ok := ctx.Value("s3Client").(*minio.Client) s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return nil, fmt.Errorf("Failed to get s3Client from context") return nil, fmt.Errorf("Failed to get s3Client from context")
@ -50,35 +49,22 @@ func copyMutation(ctx context.Context, src, dest types.ID) (*types.File, error)
} }
log.Debug("S3 'CopyObject': ", src, "-->", dest) log.Debug("S3 'CopyObject': ", src, "-->", dest)
info, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{ err := s3Client.CopyObject(ctx, src, dest)
Bucket: dest.Bucket,
Object: dest.Key,
}, minio.CopySrcOptions{
Bucket: src.Bucket,
Object: src.Key,
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
newID := types.ID{ ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, dest)
Bucket: info.Bucket,
Key: info.Key,
}
newID.Normalize()
ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, newID)
return &types.File{ return &types.File{
ID: newID, ID: dest,
}, nil }, nil
} }
func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, error) { func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, error) {
s3Client, ok := ctx.Value("s3Client").(*minio.Client) s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return nil, fmt.Errorf("Failed to get s3Client from context") return nil, fmt.Errorf("Failed to get s3Client from context")
@ -107,13 +93,7 @@ func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, er
newID.Normalize() newID.Normalize()
log.Debug("S3 'CopyObject': ", src, "-->", dest) log.Debug("S3 'CopyObject': ", src, "-->", dest)
_, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{ err := s3Client.CopyObject(ctx, file.ID, dest)
Bucket: dest.Bucket,
Object: newID.Key,
}, minio.CopySrcOptions{
Bucket: file.ID.Bucket,
Object: file.ID.Key,
})
if err != nil { if err != nil {
// TODO: handle error // TODO: handle error
@ -135,7 +115,7 @@ func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, er
} }
func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, error) { func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, error) {
s3Client, ok := ctx.Value("s3Client").(*minio.Client) s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return nil, fmt.Errorf("Failed to get s3Client from context") return nil, fmt.Errorf("Failed to get s3Client from context")
@ -150,13 +130,7 @@ func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, err
log.Debug("S3 'CopyObject': ", src, "-->", dest) log.Debug("S3 'CopyObject': ", src, "-->", dest)
// There is no (spoon) move. Only copy and delete // There is no (spoon) move. Only copy and delete
info, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{ err := s3Client.CopyObject(ctx, src, dest)
Bucket: dest.Bucket,
Object: dest.Key,
}, minio.CopySrcOptions{
Bucket: src.Bucket,
Object: src.Key,
})
if err != nil { if err != nil {
return nil, err return nil, err
@ -168,54 +142,37 @@ func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, err
return nil, err return nil, err
} }
newId := types.ID{ ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, dest)
Bucket: info.Bucket,
Key: info.Key,
}
newId.Normalize()
ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, newId)
return &types.File{ return &types.File{
ID: newId, ID: dest,
}, nil }, nil
} }
func createDirectory(ctx context.Context, id types.ID) (*types.Directory, error) { func createDirectory(ctx context.Context, id types.ID) (*types.Directory, error) {
s3Client, ok := ctx.Value("s3Client").(*minio.Client) s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return nil, fmt.Errorf("Failed to get s3Client from context") return nil, fmt.Errorf("Failed to get s3Client from context")
} }
log.Debug("S3 'PutObject': ", id) log.Debug("S3 'PutObject': ", id)
info, err := s3Client.PutObject(ctx, id.Bucket, id.Key, strings.NewReader(""), 0, minio.PutObjectOptions{ err := s3Client.PutObject(ctx, id, strings.NewReader(""), 0) // TODO: s3client interface needs content type parameter
ContentType: "application/x-directory",
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
newID := types.ID{ ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, id)
Bucket: info.Bucket,
Key: info.Key,
}
newID.Normalize()
ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, newID)
return &types.Directory{ return &types.Directory{
ID: newID, ID: id,
}, nil }, nil
} }
func deleteDirectory(ctx context.Context, id types.ID) error { func deleteDirectory(ctx context.Context, id types.ID) error {
s3Client, ok := ctx.Value("s3Client").(*minio.Client) s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return fmt.Errorf("Failed to get s3Client from context") return fmt.Errorf("Failed to get s3Client from context")
@ -236,11 +193,9 @@ func deleteDirectory(ctx context.Context, id types.ID) error {
} }
// Delete all child files // Delete all child files
var keysToDel []string
for _, file := range files { for _, file := range files {
keysToDel = append(keysToDel, file.ID.Key) s3Client.RemoveObject(ctx, file.ID)
} }
err = helper.DeleteMultiple(ctx, *s3Client, id.Bucket, keysToDel)
if err != nil { if err != nil {
return err return err
@ -253,7 +208,7 @@ func deleteDirectory(ctx context.Context, id types.ID) error {
// TODO: check if this is normal behavior when working with s3 // TODO: check if this is normal behavior when working with s3
if len(files) == 0 { if len(files) == 0 {
log.Debug("S3 'RemoveObject': ", id) log.Debug("S3 'RemoveObject': ", id)
err := s3Client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{}) err := s3Client.RemoveObject(ctx, id)
if err != nil { if err != nil {
return err return err
} }

37
internal/helper/helper.go
View File

@ -6,37 +6,15 @@ import (
"strings" "strings"
"time" "time"
"git.kapelle.org/niklas/s3browser/internal/s3"
types "git.kapelle.org/niklas/s3browser/internal/types" types "git.kapelle.org/niklas/s3browser/internal/types"
"github.com/golang-jwt/jwt" "github.com/golang-jwt/jwt"
"github.com/minio/minio-go/v7"
log "github.com/sirupsen/logrus"
) )
func GetFilenameFromKey(id string) string { func GetFilenameFromKey(id string) string {
return filepath.Base(id) return filepath.Base(id)
} }
func DeleteMultiple(ctx context.Context, s3Client minio.Client, bucket string, keys []string) error {
objectsCh := make(chan minio.ObjectInfo, 1)
go func() {
defer close(objectsCh)
for _, id := range keys {
objectsCh <- minio.ObjectInfo{
Key: id,
}
}
}()
log.Debug("S3 'RemoveObject': ", keys)
for err := range s3Client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) {
log.Error("Failed to delete object ", err.ObjectName, " because: ", err.Err.Error())
// TODO: error handel
}
return nil
}
func GetParentDir(id types.ID) types.ID { func GetParentDir(id types.ID) types.ID {
dirs := strings.Split(id.Key, "/") dirs := strings.Split(id.Key, "/")
@ -57,17 +35,10 @@ func GetParentDir(id types.ID) types.ID {
return parent return parent
} }
func ObjInfoToFile(objInfo minio.ObjectInfo, bucket string) *types.File { func ObjInfoToFile(objInfo s3.Object, bucket string) *types.File {
objID := types.ID{
Bucket: bucket,
Key: objInfo.Key,
}
objID.Normalize()
return &types.File{ return &types.File{
ID: objID, ID: objInfo.ID,
Name: GetFilenameFromKey(objID.Key), Name: objInfo.ID.Name(),
Size: objInfo.Size, Size: objInfo.Size,
ContentType: objInfo.ContentType, ContentType: objInfo.ContentType,
ETag: objInfo.ETag, ETag: objInfo.ETag,

22
internal/httpserver/httpServer.go
View File

@ -4,9 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"mime"
"net/http" "net/http"
"path/filepath"
"time" "time"
"github.com/golang-jwt/jwt" "github.com/golang-jwt/jwt"
@ -15,12 +13,12 @@ import (
"github.com/graphql-go/graphql" "github.com/graphql-go/graphql"
"github.com/graphql-go/graphql/gqlerrors" "github.com/graphql-go/graphql/gqlerrors"
"github.com/graphql-go/handler" "github.com/graphql-go/handler"
"github.com/minio/minio-go/v7"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
helper "git.kapelle.org/niklas/s3browser/internal/helper" helper "git.kapelle.org/niklas/s3browser/internal/helper"
"git.kapelle.org/niklas/s3browser/internal/loader" "git.kapelle.org/niklas/s3browser/internal/loader"
"git.kapelle.org/niklas/s3browser/internal/s3"
types "git.kapelle.org/niklas/s3browser/internal/types" types "git.kapelle.org/niklas/s3browser/internal/types"
) )
@ -112,7 +110,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
return return
} }
s3Client := ctx.Value("s3Client").(*minio.Client) s3Client := ctx.Value("s3Client").(s3.S3Service)
idString := r.URL.Query().Get("id") idString := r.URL.Query().Get("id")
id := types.ParseID(idString) id := types.ParseID(idString)
@ -124,7 +122,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
} }
log.Debug("S3 'StatObject': ", id) log.Debug("S3 'StatObject': ", id)
objInfo, err := s3Client.StatObject(context.Background(), id.Bucket, id.Key, minio.GetObjectOptions{}) objInfo, err := s3Client.StatObject(context.Background(), *id)
if err != nil { if err != nil {
log.Error("Failed to get object info: ", err) log.Error("Failed to get object info: ", err)
@ -139,7 +137,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
} }
log.Debug("S3 'GetObject': ", id) log.Debug("S3 'GetObject': ", id)
obj, err := s3Client.GetObject(context.Background(), id.Bucket, id.Key, minio.GetObjectOptions{}) obj, err := s3Client.GetObject(context.Background(), *id)
if err != nil { if err != nil {
log.Error("Failed to get object: ", err) log.Error("Failed to get object: ", err)
@ -148,7 +146,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
} }
rw.Header().Set("Cache-Control", "must-revalidate") rw.Header().Set("Cache-Control", "must-revalidate")
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key)))) rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", id.Name()))
rw.Header().Set("Content-Type", objInfo.ContentType) rw.Header().Set("Content-Type", objInfo.ContentType)
rw.Header().Set("ETag", objInfo.ETag) rw.Header().Set("ETag", objInfo.ETag)
@ -166,7 +164,7 @@ func httpPostFile(ctx context.Context, rw http.ResponseWriter, r *http.Request)
return return
} }
s3Client := ctx.Value("s3Client").(*minio.Client) s3Client := ctx.Value("s3Client").(s3.S3Service)
idString := r.URL.Query().Get("id") idString := r.URL.Query().Get("id")
@ -180,13 +178,11 @@ func httpPostFile(ctx context.Context, rw http.ResponseWriter, r *http.Request)
id.Normalize() id.Normalize()
contentType := r.Header.Get("Content-Type") // contentType := r.Header.Get("Content-Type")
mimeType, _, _ := mime.ParseMediaType(contentType) // mimeType, _, _ := mime.ParseMediaType(contentType)
log.Debug("S3 'PutObject': ", id) log.Debug("S3 'PutObject': ", id)
_, err := s3Client.PutObject(context.Background(), id.Bucket, id.Key, r.Body, r.ContentLength, minio.PutObjectOptions{ err := s3Client.PutObject(context.Background(), *id, r.Body, r.ContentLength) // TODO: put content type
ContentType: mimeType,
})
if err != nil { if err != nil {
rw.WriteHeader(http.StatusInternalServerError) rw.WriteHeader(http.StatusInternalServerError)

50
internal/loader/batch.go
View File

@ -4,9 +4,9 @@ import (
"context" "context"
"fmt" "fmt"
types "git.kapelle.org/niklas/s3browser/internal/types" "git.kapelle.org/niklas/s3browser/internal/s3"
"git.kapelle.org/niklas/s3browser/internal/types"
"github.com/graph-gophers/dataloader" "github.com/graph-gophers/dataloader"
"github.com/minio/minio-go/v7"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
@ -14,7 +14,7 @@ import (
func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result { func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
var results []*dataloader.Result var results []*dataloader.Result
s3Client, ok := c.Value("s3Client").(*minio.Client) s3Client, ok := c.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@ -22,11 +22,19 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
for _, v := range k { for _, v := range k {
id := v.Raw().(types.ID) id := v.Raw().(types.ID)
objects, err := s3Client.ListObjects(c, id)
if err != nil {
results = append(results, &dataloader.Result{ results = append(results, &dataloader.Result{
Data: listObjects(s3Client, id, false), Data: nil,
Error: err,
})
} else {
results = append(results, &dataloader.Result{
Data: objects,
Error: nil, Error: nil,
}) })
} }
}
return results return results
} }
@ -35,7 +43,7 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataloader.Result { func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
var results []*dataloader.Result var results []*dataloader.Result
s3Client, ok := c.Value("s3Client").(*minio.Client) s3Client, ok := c.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@ -43,35 +51,27 @@ func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataload
for _, v := range k { for _, v := range k {
id := v.Raw().(types.ID) id := v.Raw().(types.ID)
objects, err := s3Client.ListObjectsRecursive(c, id)
if err != nil {
results = append(results, &dataloader.Result{ results = append(results, &dataloader.Result{
Data: listObjects(s3Client, id, true), Data: nil,
Error: err,
})
} else {
results = append(results, &dataloader.Result{
Data: objects,
Error: nil, Error: nil,
}) })
} }
}
return results return results
} }
// listObjects helper func for listObjectsBatch
func listObjects(s3Client *minio.Client, id types.ID, recursive bool) []minio.ObjectInfo {
log.Debug("S3 'ListObjects': ", id)
objectCh := s3Client.ListObjects(context.Background(), id.Bucket, minio.ListObjectsOptions{
Prefix: id.Key,
Recursive: recursive,
})
result := make([]minio.ObjectInfo, 0)
for obj := range objectCh {
result = append(result, obj)
}
return result
}
func listBucketsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result { func listBucketsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
var results []*dataloader.Result var results []*dataloader.Result
s3Client, ok := c.Value("s3Client").(*minio.Client) s3Client, ok := c.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@ -100,7 +100,7 @@ func statObjectBatch(ctx context.Context, k dataloader.Keys) []*dataloader.Resul
log.Debug("statObjectBatch") log.Debug("statObjectBatch")
var results []*dataloader.Result var results []*dataloader.Result
s3Client, ok := ctx.Value("s3Client").(*minio.Client) s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
if !ok { if !ok {
return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context")) return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@ -109,7 +109,7 @@ func statObjectBatch(ctx context.Context, k dataloader.Keys) []*dataloader.Resul
for _, v := range k { for _, v := range k {
id := v.Raw().(types.ID) id := v.Raw().(types.ID)
log.Debug("S3 'StatObject': ", id) log.Debug("S3 'StatObject': ", id)
stat, err := s3Client.StatObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{}) stat, err := s3Client.StatObject(ctx, id)
results = append(results, &dataloader.Result{ results = append(results, &dataloader.Result{
Data: stat, Data: stat,
Error: err, Error: err,

40
internal/loader/loader.go
View File

@ -3,14 +3,12 @@ package loader
import ( import (
"context" "context"
"fmt" "fmt"
"path/filepath"
"strings"
"git.kapelle.org/niklas/s3browser/internal/cache" "git.kapelle.org/niklas/s3browser/internal/cache"
"git.kapelle.org/niklas/s3browser/internal/helper" "git.kapelle.org/niklas/s3browser/internal/helper"
"git.kapelle.org/niklas/s3browser/internal/s3"
types "git.kapelle.org/niklas/s3browser/internal/types" types "git.kapelle.org/niklas/s3browser/internal/types"
"github.com/graph-gophers/dataloader" "github.com/graph-gophers/dataloader"
"github.com/minio/minio-go/v7"
) )
type Loader struct { type Loader struct {
@ -68,10 +66,8 @@ func (l *Loader) GetFiles(ctx context.Context, path types.ID) ([]types.File, err
var files []types.File var files []types.File
for _, obj := range objects.([]minio.ObjectInfo) { for _, obj := range objects.([]s3.Object) {
if obj.Err != nil { if !obj.ID.IsDirectory() {
return nil, obj.Err
} else if !strings.HasSuffix(obj.Key, "/") {
files = append(files, *helper.ObjInfoToFile(obj, path.Bucket)) files = append(files, *helper.ObjInfoToFile(obj, path.Bucket))
} }
} }
@ -86,13 +82,13 @@ func (l *Loader) GetFile(ctx context.Context, id types.ID) (*types.File, error)
if err != nil { if err != nil {
return nil, err return nil, err
} }
objInfo, ok := result.(minio.ObjectInfo) objInfo, ok := result.(*s3.Object)
if !ok { if !ok {
return nil, fmt.Errorf("Failed to stats object") return nil, fmt.Errorf("Failed to stats object")
} }
return helper.ObjInfoToFile(objInfo, id.Bucket), nil return helper.ObjInfoToFile(*objInfo, id.Bucket), nil
} }
func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory, error) { func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory, error) {
@ -104,20 +100,10 @@ func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory,
} }
var dirs []types.Directory var dirs []types.Directory
for _, obj := range result.([]minio.ObjectInfo) { for _, obj := range result.([]s3.Object) {
if obj.Err != nil { if obj.ID.IsDirectory() {
return nil, obj.Err
} else if strings.HasSuffix(obj.Key, "/") {
resultID := types.ID{
Bucket: path.Bucket,
Key: obj.Key,
}
resultID.Normalize()
dirs = append(dirs, types.Directory{ dirs = append(dirs, types.Directory{
ID: resultID, ID: obj.ID,
Name: filepath.Base(obj.Key),
}) })
} }
} }
@ -134,13 +120,7 @@ func (l *Loader) GetBuckets(ctx context.Context) ([]string, error) {
return nil, err return nil, err
} }
bucketsInfo := result.([]minio.BucketInfo) return result.([]string), nil
var buckets []string
for _, i := range bucketsInfo {
buckets = append(buckets, i.Name)
}
return buckets, nil
} }
func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.File, error) { func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.File, error) {
@ -151,7 +131,7 @@ func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.
return nil, err return nil, err
} }
objects := result.([]minio.ObjectInfo) objects := result.([]s3.Object)
var files []types.File var files []types.File
for _, obj := range objects { for _, obj := range objects {

81
internal/s3/minio.go
View File

@ -28,44 +28,62 @@ func NewMinio(config types.AppConfig) (S3Service, error) {
}, nil }, nil
} }
func (m *minioS3) ListBuckets(ctx context.Context) ([]Bucket, error) { func (m *minioS3) ListBuckets(ctx context.Context) ([]string, error) {
buckets, err := m.client.ListBuckets(ctx) buckets, err := m.client.ListBuckets(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var rtn []Bucket var rtn []string
for _, v := range buckets { for _, v := range buckets {
rtn = append(rtn, Bucket(v.Name)) rtn = append(rtn, v.Name)
} }
return rtn, nil return rtn, nil
} }
func (m *minioS3) ListObjects(ctx context.Context, id types.ID) ([]types.File, []types.Directory, error) { func (m *minioS3) ListObjects(ctx context.Context, id types.ID) ([]Object, error) {
var files []types.File var result []Object
var dirs []types.Directory
for objInfo := range m.client.ListObjects(ctx, id.Bucket, minio.ListObjectsOptions{}) { for objInfo := range m.client.ListObjects(ctx, id.Bucket, minio.ListObjectsOptions{
Prefix: id.Key,
Recursive: false,
}) {
objId := types.ID{ objId := types.ID{
Bucket: id.Bucket, Bucket: id.Bucket,
Key: objInfo.Key, Key: objInfo.Key,
} }
objId.Normalize() result = append(result, Object{
ID: objId,
if objId.IsDirectory() { Size: objInfo.Size,
dirs = append(dirs, *obkInfoToDir(objInfo, objId)) })
} else {
files = append(files, *objInfoToFile(objInfo, objId))
}
} }
return files, dirs, nil return result, nil
} }
func (m *minioS3) GetObject(ctx context.Context, id types.ID) (Object, error) { func (m *minioS3) ListObjectsRecursive(ctx context.Context, id types.ID) ([]Object, error) {
var result []Object
for objInfo := range m.client.ListObjects(ctx, id.Bucket, minio.ListObjectsOptions{
Prefix: id.Key,
Recursive: true,
}) {
objId := types.ID{
Bucket: id.Bucket,
Key: objInfo.Key,
}
result = append(result, Object{
ID: objId,
Size: objInfo.Size,
})
}
return result, nil
}
func (m *minioS3) GetObject(ctx context.Context, id types.ID) (ObjectReader, error) {
object, err := m.client.GetObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{}) object, err := m.client.GetObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{})
if err != nil { if err != nil {
@ -92,35 +110,22 @@ func (m *minioS3) CopyObject(ctx context.Context, src types.ID, dest types.ID) e
return err return err
} }
func (m *minioS3) StatObject(ctx context.Context, id types.ID) (*types.File, error) { func (m *minioS3) StatObject(ctx context.Context, id types.ID) (*Object, error) {
info, err := m.client.StatObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{}) info, err := m.client.StatObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{})
if err != nil { if err != nil {
return nil, err return nil, err
} }
return objInfoToFile(info, id), nil return &Object{
ID: id,
Size: info.Size,
LastModified: info.LastModified,
ContentType: info.ContentType,
ETag: info.ETag,
}, nil
} }
func (m *minioS3) RemoveObject(ctx context.Context, id types.ID) error { func (m *minioS3) RemoveObject(ctx context.Context, id types.ID) error {
return m.client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{}) return m.client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{})
} }
func objInfoToFile(objInfo minio.ObjectInfo, id types.ID) *types.File {
return &types.File{
ID: id,
Name: id.Name(),
Size: objInfo.Size,
ContentType: objInfo.ContentType,
ETag: objInfo.ETag,
LastModified: objInfo.LastModified,
}
}
func obkInfoToDir(objInfo minio.ObjectInfo, id types.ID) *types.Directory {
return &types.Directory{
ID: id,
Name: id.Name(),
}
}

57
internal/s3/mock.go
View File

@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"strings"
"time" "time"
"git.kapelle.org/niklas/s3browser/internal/types" "git.kapelle.org/niklas/s3browser/internal/types"
@ -39,35 +40,41 @@ func NewMockS3(buckets []string) (S3Service, error) {
}, nil }, nil
} }
func (m *mockS3) ListBuckets(ctx context.Context) ([]Bucket, error) { func (m *mockS3) ListBuckets(ctx context.Context) ([]string, error) {
var rtn []Bucket return m.buckets, nil
for _, v := range m.buckets {
rtn = append(rtn, Bucket(v))
}
return rtn, nil
} }
func (m *mockS3) ListObjects(ctx context.Context, id types.ID) ([]types.File, []types.Directory, error) { func (m *mockS3) ListObjects(ctx context.Context, id types.ID) ([]Object, error) {
var files []types.File var results []Object
var dirs []types.Directory
for k, v := range m.objects { for k, v := range m.objects {
if k.Bucket == id.Bucket { if k.Bucket == id.Bucket {
if k.Parent().Key == id.Key { if k.Parent().Key == id.Key {
if k.IsDirectory() { results = append(results, *mockObjToObject(v, k))
dirs = append(dirs, *mockObjToDir(v, k))
} else {
files = append(files, *mockObjToFile(v, k))
}
} }
} }
} }
return files, dirs, nil return results, nil
} }
func (m *mockS3) GetObject(ctx context.Context, id types.ID) (Object, error) { func (m *mockS3) ListObjectsRecursive(ctx context.Context, id types.ID) ([]Object, error) {
var results []Object
for k, v := range m.objects {
if k.Bucket == id.Bucket {
if strings.HasPrefix(k.Key, id.Key) {
if k.Parent().Key == id.Key {
results = append(results, *mockObjToObject(v, k))
}
}
}
}
return results, nil
}
func (m *mockS3) GetObject(ctx context.Context, id types.ID) (ObjectReader, error) {
mockObj, exist := m.objects[id] mockObj, exist := m.objects[id]
if !exist { if !exist {
@ -107,14 +114,14 @@ func (m *mockS3) CopyObject(ctx context.Context, src types.ID, dest types.ID) er
return nil return nil
} }
func (m *mockS3) StatObject(ctx context.Context, id types.ID) (*types.File, error) { func (m *mockS3) StatObject(ctx context.Context, id types.ID) (*Object, error) {
mockObj, exist := m.objects[id] mockObj, exist := m.objects[id]
if !exist { if !exist {
return nil, fmt.Errorf("Object not found") return nil, fmt.Errorf("Object not found")
} }
return mockObjToFile(mockObj, id), nil return mockObjToObject(mockObj, id), nil
} }
func (m *mockS3) RemoveObject(ctx context.Context, id types.ID) error { func (m *mockS3) RemoveObject(ctx context.Context, id types.ID) error {
@ -122,20 +129,12 @@ func (m *mockS3) RemoveObject(ctx context.Context, id types.ID) error {
return nil return nil
} }
func mockObjToFile(mockObj mockObject, id types.ID) *types.File { func mockObjToObject(mockObj mockObject, id types.ID) *Object {
return &types.File{ return &Object{
ID: id, ID: id,
Name: id.Name(),
Size: int64(len(mockObj.content)), Size: int64(len(mockObj.content)),
ContentType: mockObj.contentType, ContentType: mockObj.contentType,
LastModified: mockObj.lastMod, LastModified: mockObj.lastMod,
ETag: fmt.Sprintf("%x", md5.Sum(mockObj.content)), ETag: fmt.Sprintf("%x", md5.Sum(mockObj.content)),
} }
} }
func mockObjToDir(mockObj mockObject, id types.ID) *types.Directory {
return &types.Directory{
ID: id,
Name: id.Name(),
}
}

24
internal/s3/s3.go
View File

@ -3,27 +3,35 @@ package s3
import ( import (
"context" "context"
"io" "io"
"time"
"git.kapelle.org/niklas/s3browser/internal/types" "git.kapelle.org/niklas/s3browser/internal/types"
) )
type Bucket string type ObjectReader interface {
type Object interface {
io.Reader io.Reader
io.Seeker io.Seeker
io.ReaderAt io.ReaderAt
io.Closer io.Closer
} }
type S3Service interface { type Object struct {
ListBuckets(ctx context.Context) ([]Bucket, error) ID types.ID
Size int64
LastModified time.Time
ContentType string
ETag string
}
GetObject(ctx context.Context, id types.ID) (Object, error) type S3Service interface {
ListBuckets(ctx context.Context) ([]string, error)
GetObject(ctx context.Context, id types.ID) (ObjectReader, error)
PutObject(ctx context.Context, id types.ID, reader io.Reader, objectSize int64) error PutObject(ctx context.Context, id types.ID, reader io.Reader, objectSize int64) error
ListObjects(ctx context.Context, id types.ID) ([]types.File, []types.Directory, error) ListObjects(ctx context.Context, id types.ID) ([]Object, error)
ListObjectsRecursive(ctx context.Context, id types.ID) ([]Object, error)
CopyObject(ctx context.Context, src types.ID, dest types.ID) error CopyObject(ctx context.Context, src types.ID, dest types.ID) error
StatObject(ctx context.Context, id types.ID) (*types.File, error) StatObject(ctx context.Context, id types.ID) (*Object, error)
RemoveObject(ctx context.Context, id types.ID) error RemoveObject(ctx context.Context, id types.ID) error
} }

13
internal/s3Broswer.go
View File

@ -3,25 +3,16 @@ package s3browser
import ( import (
"context" "context"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"git.kapelle.org/niklas/s3browser/internal/db" "git.kapelle.org/niklas/s3browser/internal/db"
gql "git.kapelle.org/niklas/s3browser/internal/gql" gql "git.kapelle.org/niklas/s3browser/internal/gql"
httpserver "git.kapelle.org/niklas/s3browser/internal/httpserver" httpserver "git.kapelle.org/niklas/s3browser/internal/httpserver"
"git.kapelle.org/niklas/s3browser/internal/loader" "git.kapelle.org/niklas/s3browser/internal/loader"
"git.kapelle.org/niklas/s3browser/internal/s3"
types "git.kapelle.org/niklas/s3browser/internal/types" types "git.kapelle.org/niklas/s3browser/internal/types"
) )
// setupS3Client connect the s3Client
func setupS3Client(config types.AppConfig) (*minio.Client, error) {
return minio.New(config.S3Endoint, &minio.Options{
Creds: credentials.NewStaticV4(config.S3AccessKey, config.S3SecretKey, ""),
Secure: config.S3SSL,
})
}
// Start starts the app // Start starts the app
func Start(config types.AppConfig) { func Start(config types.AppConfig) {
@ -30,7 +21,7 @@ func Start(config types.AppConfig) {
} }
log.Info("Starting") log.Info("Starting")
s3Client, err := setupS3Client(config) s3Client, err := s3.NewMinio(config)
if err != nil { if err != nil {
log.Error("Failed to setup s3 client: ", err.Error()) log.Error("Failed to setup s3 client: ", err.Error())